Skip to content

6 more naive methods for Tensor Primitives. #92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 16, 2023
Merged

6 more naive methods for Tensor Primitives. #92142

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
651d230
6 more naive methods
michaelgsharp Sep 15, 2023
75769c4
updates from pr comments
michaelgsharp Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
public static class TensorPrimitives
public static partial class TensorPrimitives
{
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float Normalize(System.ReadOnlySpan<float> x) { throw null; }
public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
}
}
133 changes: 133 additions & 0 deletions src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.ComponentModel;

namespace System.Numerics.Tensors
{
/// <summary>Performs primitive tensor operations over spans of memory.</summary>
Expand Down Expand Up @@ -253,5 +255,136 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
destination[i] = MathF.Tanh(x[i]);
}
}

/// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The cosine similarity between the two vectors.</returns>
public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}

var dotprod = 0f;
var magx = 0f;
var magy = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
magx += MathF.Pow(x[i], 2);
magy += MathF.Pow(y[i], 2);
}

return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
}

/// <summary>
/// Compute the distance between two points in Euclidean space.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The Euclidean distance.</returns>
public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}

var distance = 0f;

for (int i = 0; i < x.Length; i++)
{
distance += MathF.Pow(x[i] - y[i], 2);
}

return MathF.Sqrt(distance);
}

/// <summary>
/// A mathematical operation that takes two vectors and returns a scalar.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The dot product.</returns>
public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}

var dotprod = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
}

return dotprod;
}

/// <summary>
/// A mathematical operation that takes a vector and returns the L2 norm.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <returns>The L2 norm.</returns>
public static float Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
{
var magx = 0f;

for (int i = 0; i < x.Length; i++)
{
magx += MathF.Pow(x[i], 2);
}

return MathF.Sqrt(magx);
}

/// <summary>
/// A function that takes a collection of real numbers and returns a probability distribution.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}

var expSum = 0f;

for (int i = 0; i < x.Length; i++)
{
expSum += MathF.Pow((float)Math.E, x[i]);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?

Why (float)Math.E and not MathF.E?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.

}

for (int i = 0; i < destination.Length; i++)
{
destination[i] = MathF.Exp(x[i]) / expSum;
}
}

/// <summary>
/// A function that takes a real number and returns a value between 0 and 1.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}

for (int i = 0; i < x.Length; i++)
{
destination[i] = 1f / (1 + MathF.Exp(-x[i]));
}
}
}
}
118 changes: 118 additions & 0 deletions src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,5 +702,123 @@ public static void TanhTensor_ThrowsForTooShortDestination(int tensorLength)

AssertExtensions.Throws<ArgumentException>("destination", () => TensorPrimitives.Tanh(x, destination));
}

[Theory]
[MemberData(nameof(TensorLengths))]
public static void CosineSimilarity_ThrowsForMismatchedLengths_x_y(int tensorLength)
{
float[] x = CreateAndFillTensor(tensorLength);
float[] y = CreateAndFillTensor(tensorLength - 1);

Assert.Throws<ArgumentException>(() => TensorPrimitives.CosineSimilarity(x, y));
}

[Theory]
[InlineData(new float[] { 3, 2, 0, 5 }, new float[] { 1, 0, 0, 0 }, 0.49f)]
[InlineData(new float[] { 1, 1, 1, 1, 1, 0 }, new float[] { 1, 1, 1, 1, 0, 1 }, 0.80f)]
public static void CosineSimilarity(float[] x, float[] y, float expectedResult)
{
Assert.Equal(expectedResult, TensorPrimitives.CosineSimilarity(x, y), .01f);
}

[Theory]
[MemberData(nameof(TensorLengths))]
public static void Distance_ThrowsForMismatchedLengths_x_y(int tensorLength)
{
float[] x = CreateAndFillTensor(tensorLength);
float[] y = CreateAndFillTensor(tensorLength - 1);

Assert.Throws<ArgumentException>(() => TensorPrimitives.Distance(x, y));
}

[Theory]
[InlineData(new float[] { 3, 2 }, new float[] { 4, 1 }, 1.4142f)]
[InlineData(new float[] { 0, 4 }, new float[] { 6, 2 }, 6.3245f)]
[InlineData(new float[] { 1, 2, 3 }, new float[] { 4, 5, 6 }, 5.1961f)]
[InlineData(new float[] { 5, 1, 6, 10 }, new float[] { 7, 2, 8, 4 }, 6.7082f)]
public static void Distance(float[] x, float[] y, float expectedResult)
{
Assert.Equal(expectedResult, TensorPrimitives.Distance(x, y), .001f);
}

[Theory]
[MemberData(nameof(TensorLengths))]
public static void Dot_ThrowsForMismatchedLengths_x_y(int tensorLength)
{
float[] x = CreateAndFillTensor(tensorLength);
float[] y = CreateAndFillTensor(tensorLength - 1);

Assert.Throws<ArgumentException>(() => TensorPrimitives.Dot(x, y));
}

[Theory]
[InlineData(new float[] { 1, 3, -5 }, new float[] { 4, -2, -1 }, 3)]
[InlineData(new float[] { 1, 2, 3 }, new float[] { 4, 5, 6 }, 32)]
[InlineData(new float[] { 1, 2, 3, 10, 8 }, new float[] { 4, 5, 6, -2, 7 }, 68)]
public static void Dot(float[] x, float[] y, float expectedResult)
{
Assert.Equal(expectedResult, TensorPrimitives.Dot(x, y), .001f);
}

[Theory]
[InlineData(new float[] { 1, 2, 3 }, 3.7416)]
[InlineData(new float[] { 3, 4 }, 5)]
[InlineData(new float[] { 3 }, 3)]
[InlineData(new float[] { 3, 4, 1, 2 }, 5.477)]
public static void Normalize(float[] x, float expectedResult)
{
Assert.Equal(expectedResult, TensorPrimitives.Normalize(x), .001f);
}

[Theory]
[MemberData(nameof(TensorLengths))]
public static void SoftMax_ThrowsForTooShortDestination(int tensorLength)
{
float[] x = CreateAndFillTensor(tensorLength);
float[] destination = CreateTensor(tensorLength - 1);

AssertExtensions.Throws<ArgumentException>("destination", () => TensorPrimitives.SoftMax(x, destination));
}

[Theory]
[InlineData(new float[] { 3, 1, .2f }, new float[] { 0.8360188f, 0.11314284f, 0.05083836f })]
[InlineData(new float[] { 3, 4, 1 }, new float[] { 0.2594f, 0.7052f, 0.0351f })]
[InlineData(new float[] { 5, 3 }, new float[] { 0.8807f, 0.1192f })]
[InlineData(new float[] { 4, 2, 1, 9 }, new float[] { 0.0066f, 9.04658e-4f, 3.32805e-4f, 0.9920f})]
public static void SoftMax(float[] x, float[] expectedResult)
{
var dest = new float[x.Length];
TensorPrimitives.SoftMax(x, dest);

for (int i = 0; i < dest.Length; i++)
{
Assert.Equal(expectedResult[i], dest[i], .001f);
}
}

[Theory]
[MemberData(nameof(TensorLengths))]
public static void Sigmoid_ThrowsForTooShortDestination(int tensorLength)
{
float[] x = CreateAndFillTensor(tensorLength);
float[] destination = CreateTensor(tensorLength - 1);

AssertExtensions.Throws<ArgumentException>("destination", () => TensorPrimitives.Sigmoid(x, destination));
}

[Theory]
[InlineData(new float[] { -5, -4.5f, -4 }, new float[] { 0.0066f, 0.0109f, 0.0179f })]
[InlineData(new float[] { 4.5f, 5 }, new float[] { 0.9890f, 0.9933f })]
[InlineData(new float[] { 0, -3, 3, .5f }, new float[] { 0.5f, 0.0474f, 0.9525f, 0.6224f })]
public static void Sigmoid(float[] x, float[] expectedResult)
{
var dest = new float[x.Length];
TensorPrimitives.Sigmoid(x, dest);

for (int i = 0; i < dest.Length; i++)
{
Assert.Equal(expectedResult[i], dest[i], .001f);
}
}
}
}