Skip to content

6 more naive methods for Tensor Primitives. #92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 16, 2023
Merged

6 more naive methods for Tensor Primitives. #92142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
651d230
6 more naive methods
michaelgsharp Sep 15, 2023
75769c4
updates from pr comments
michaelgsharp Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
public static class TensorPrimitives
public static partial class TensorPrimitives
{
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
}
}
5 changes: 4 additions & 1 deletion src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,10 @@
<data name="Argument_DestinationTooShort" xml:space="preserve">
<value>Destination is too short.</value>
</data>
<data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
<value>Input span arguments must not be empty.</value>
</data>
<data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
<value>Input span arguments must all have the same length.</value>
</data>
</root>
</root>
157 changes: 157 additions & 0 deletions src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
destination[i] = MathF.Tanh(x[i]);
}
}

/// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The cosine similarity between the two vectors.</returns>
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}
if (x.Length == 0 || y.Length == 0)
Copy link
Member

@tannergooding tannergooding Sep 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.

That being said, is it desirable to throw here rather than allowing it to just return NaN?

{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float dotprod = 0f;
float magx = 0f;
float magy = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
magx += x[i] * x[i];
magy += y[i] * y[i];
}

return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
}

/// <summary>
/// Compute the distance between two points in Euclidean space.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The Euclidean distance.</returns>
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}
if (x.Length == 0 || y.Length == 0)
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float distance = 0f;

for (int i = 0; i < x.Length; i++)
{
float dist = x[i] - y[i];
distance += dist * dist;
}

return MathF.Sqrt(distance);
}

/// <summary>
/// A mathematical operation that takes two vectors and returns a scalar.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The dot product.</returns>
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}

float dotprod = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
}

return dotprod;
}

/// <summary>
/// A mathematical operation that takes a vector and returns the L2 norm.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <returns>The L2 norm.</returns>
public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
Copy link
Member

@tannergooding tannergooding Sep 18, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.

The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())

{
float magx = 0f;

for (int i = 0; i < x.Length; i++)
{
magx += x[i] * x[i];
}

return MathF.Sqrt(magx);
}

/// <summary>
/// A function that takes a collection of real numbers and returns a probability distribution.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
/// <exception cref="ArgumentException">Destination is too short.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}
if (x.Length == 0)
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float expSum = 0f;

for (int i = 0; i < x.Length; i++)
{
expSum += MathF.Pow((float)Math.E, x[i]);
Copy link
Member

@tannergooding tannergooding Sep 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?

Why (float)Math.E and not MathF.E?

Copy link
Member

@stephentoub stephentoub Sep 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.

}

for (int i = 0; i < x.Length; i++)
{
destination[i] = MathF.Exp(x[i]) / expSum;
}
}

/// <summary>
/// A function that takes a real number and returns a value between 0 and 1.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
/// <exception cref="ArgumentException">Destination is too short.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}
if (x.Length == 0)
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

for (int i = 0; i < x.Length; i++)
{
destination[i] = 1f / (1 + MathF.Exp(-x[i]));
}
}
}
}
4 changes: 4 additions & 0 deletions src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
[DoesNotReturn]
public static void ThrowArgument_SpansMustHaveSameLength() =>
throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

[DoesNotReturn]
public static void ThrowArgument_SpansMustBeNonEmpty() =>
throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
}
}
Loading