Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support the neural query type and text_embedding ingest processor type #636

Merged
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Improve test
Signed-off-by: Thomas Farr <[email protected]>
  • Loading branch information
Xtansia committed Aug 22, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 6050c7659cfc0f4ca4d00810e6918dd25c8fabe6
Original file line number Diff line number Diff line change
@@ -75,7 +75,7 @@ public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures
/// This can be useful to fail early when subsequent operations are relying on installation
/// succeeding.
/// </summary>
public bool ValidatePluginsToInstall { get; } = true;
public bool ValidatePluginsToInstall { get; set; } = true;

public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL);

Original file line number Diff line number Diff line change
@@ -61,38 +61,37 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
.Where(p => !p.IsValid(v))
.Select(p => p.SubProductName).ToList();
if (invalidPlugins.Any())
throw new OpenSearchCleanExitException(
$"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} ");
}
{
throw new OpenSearchCleanExitException(
$"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} ");
}
}

foreach (var plugin in requiredPlugins)
{
var includedByDefault = plugin.IsIncludedOutOfTheBox(v);
if (includedByDefault)
if (plugin.IsIncludedOutOfTheBox(v))
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}");
continue;
}

var validForCurrentVersion = plugin.IsValid(v);
if (!validForCurrentVersion)
if (!plugin.IsValid(v))
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}");
continue;
}

var alreadyInstalled = AlreadyInstalled(fs, plugin.SubProductName);
if (alreadyInstalled)
if (AlreadyInstalled(fs, plugin.SubProductName))
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] already installed");
continue;
}

cluster.Writer?.WriteDiagnostic(
$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}");
$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}");

var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config");

Original file line number Diff line number Diff line change
@@ -35,102 +35,109 @@
using Xunit;
using Xunit.Abstractions;
using Xunit.Sdk;
using Enumerable = System.Linq.Enumerable;

namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing
namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;

/// <summary>
/// A Xunit test that should be skipped, and a reason why.
/// </summary>
public abstract class SkipTestAttributeBase : Attribute
{
/// <summary>
/// An Xunit test that should be skipped, and a reason why.
/// </summary>
public abstract class SkipTestAttributeBase : Attribute
{
/// <summary>
/// Whether the test should be skipped
/// </summary>
public abstract bool Skip { get; }
/// <summary>
/// Whether the test should be skipped
/// </summary>
public abstract bool Skip { get; }

/// <summary>
/// The reason why the test should be skipped
/// </summary>
public abstract string Reason { get; }
}

/// <summary>
/// The reason why the test should be skipped
/// </summary>
public abstract string Reason { get; }
}
/// <summary>
/// An Xunit integration test
/// </summary>
[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer",
"OpenSearch.OpenSearch.Xunit")]
public class I : FactAttribute
{
}

/// <summary>
/// An Xunit integration test
/// </summary>
[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer",
"OpenSearch.OpenSearch.Xunit")]
public class I : FactAttribute
{
}
/// <summary>
/// A test discoverer used to discover integration tests cases attached
/// to test methods that are attributed with <see cref="I" /> attribute
/// </summary>
public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer
{
public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
{
}

/// <summary>
/// A test discoverer used to discover integration tests cases attached
/// to test methods that are attributed with <see cref="I" /> attribute
/// </summary>
public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer
{
public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
{
}
/// <inheritdoc />
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
out string skipReason)
{
skipReason = null;
var runIntegrationTests =
discoveryOptions.GetValue<bool>(nameof(OpenSearchXunitRunOptions.RunIntegrationTests));
if (!runIntegrationTests) return true;

/// <inheritdoc />
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
out string skipReason)
{
skipReason = null;
var runIntegrationTests =
discoveryOptions.GetValue<bool>(nameof(OpenSearchXunitRunOptions.RunIntegrationTests));
if (!runIntegrationTests) return true;
var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class);
if (cluster == null)
{
skipReason +=
$"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}";
return true;
}

var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class);
if (cluster == null)
{
skipReason +=
$"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}";
return true;
}
var openSearchVersion =
discoveryOptions.GetValue<OpenSearchVersion>(nameof(OpenSearchXunitRunOptions.Version));

var openSearchVersion =
discoveryOptions.GetValue<OpenSearchVersion>(nameof(OpenSearchXunitRunOptions.Version));
// Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges)
var skipVersionAttribute = GetAttributes<SkipVersionAttribute>(testMethod).FirstOrDefault();
if (skipVersionAttribute != null)
{
var skipVersionRanges =
skipVersionAttribute.GetNamedArgument<IList<Range>>(nameof(SkipVersionAttribute.Ranges)) ??
new List<Range>();
if (openSearchVersion == null && skipVersionRanges.Count > 0)
{
skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " +
$"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}";
return true;
}

// Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges)
var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes<SkipVersionAttribute>(testMethod));
if (skipVersionAttribute != null)
{
var skipVersionRanges =
skipVersionAttribute.GetNamedArgument<IList<Range>>(nameof(SkipVersionAttribute.Ranges)) ??
new List<Range>();
if (openSearchVersion == null && skipVersionRanges.Count > 0)
{
skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " +
$"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}";
return true;
}
if (openSearchVersion != null)
{
var reason = skipVersionAttribute.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
foreach (var range in skipVersionRanges)
{
// inrange takes prereleases into account
if (!openSearchVersion.InRange(range)) continue;
skipReason =
$"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies";
if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
return true;
}
}
}

if (openSearchVersion != null)
{
var reason = skipVersionAttribute.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
for (var index = 0; index < skipVersionRanges.Count; index++)
{
var range = skipVersionRanges[index];
// inrange takes prereleases into account
if (!openSearchVersion.InRange(range)) continue;
skipReason =
$"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies";
if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
return true;
}
}
}
// Skip if a prerelease version and has SkipPrereleaseVersionsAttribute
var skipPrerelease = GetAttributes<SkipPrereleaseVersionsAttribute>(testMethod).FirstOrDefault();
if (openSearchVersion != null && openSearchVersion.IsPreRelease && skipPrerelease != null)
{
skipReason = $"{nameof(SkipPrereleaseVersionsAttribute)} has been applied to this test";
var reason = skipPrerelease.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
return true;
}

var skipTests = GetAttributes<SkipTestAttributeBase>(testMethod)
.FirstOrDefault(a => a.GetNamedArgument<bool>(nameof(SkipTestAttributeBase.Skip)));
var skipTests = GetAttributes<SkipTestAttributeBase>(testMethod)
.FirstOrDefault(a => a.GetNamedArgument<bool>(nameof(SkipTestAttributeBase.Skip)));

if (skipTests == null) return false;
if (skipTests == null) return false;

skipReason = skipTests.GetNamedArgument<string>(nameof(SkipTestAttributeBase.Reason));
return true;
}
}
skipReason = skipTests.GetNamedArgument<string>(nameof(SkipTestAttributeBase.Reason));
return true;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

using System;

namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;

/// <summary>
/// A Xunit test that should be skipped for prerelease OpenSearch versions, and a reason why.
/// </summary>
public class SkipPrereleaseVersionsAttribute : Attribute
{
public SkipPrereleaseVersionsAttribute(string reason) => Reason = reason;

/// <summary>
/// The reason why the test should be skipped
/// </summary>
public string Reason { get; }
}
Original file line number Diff line number Diff line change
@@ -31,35 +31,34 @@
using System.Linq;
using SemanticVersioning;

namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing
namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;

/// <summary>
/// A Xunit test that should be skipped for given OpenSearch versions, and a reason why.
/// </summary>
public class SkipVersionAttribute : Attribute
{
/// <summary>
/// An Xunit test that should be skipped for given OpenSearch versions, and a reason why.
/// </summary>
public class SkipVersionAttribute : Attribute
{
// ReSharper disable once UnusedParameter.Local
// reason is used to allow the test its used on to self document why its been put in place
public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason)
{
Reason = reason;
Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma)
? new List<Range>()
: skipVersionRangesSeparatedByComma.Split(',')
.Select(r => r.Trim())
.Where(r => !string.IsNullOrWhiteSpace(r))
.Select(r => new Range(r))
.ToList();
}
// ReSharper disable once UnusedParameter.Local
// reason is used to allow the test its used on to self document why its been put in place
public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason)
{
Reason = reason;
Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma)
? new List<Range>()
: skipVersionRangesSeparatedByComma.Split(',')
.Select(r => r.Trim())
.Where(r => !string.IsNullOrWhiteSpace(r))
.Select(r => new Range(r))
.ToList();
}

/// <summary>
/// The reason why the test should be skipped
/// </summary>
public string Reason { get; }
/// <summary>
/// The reason why the test should be skipped
/// </summary>
public string Reason { get; }

/// <summary>
/// The version ranges for which the test should be skipped
/// </summary>
public IList<Range> Ranges { get; }
}
/// <summary>
/// The version ranges for which the test should be skipped
/// </summary>
public IList<Range> Ranges { get; }
}
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@
*/

using System;
using Version = SemanticVersioning.Version;

namespace OpenSearch.Stack.ArtifactsApi.Products
{
@@ -81,5 +82,9 @@ public OpenSearchPlugin(string plugin, Func<OpenSearchVersion, bool> isValid = n
public static OpenSearchPlugin DeleteByQuery { get; } = new("delete-by-query", version => version < "1.0.0");

public static OpenSearchPlugin Knn { get; } = new("opensearch-knn");
}

public static OpenSearchPlugin MachineLearning { get; } = new("opensearch-ml", v => v.BaseVersion() >= new Version("1.3.0") && !v.IsPreRelease);

public static OpenSearchPlugin NeuralSearch { get; } = new("opensearch-neural-search", v => v.BaseVersion() >= new Version("2.4.0") && !v.IsPreRelease);
}
}
1 change: 0 additions & 1 deletion samples/Samples/NeuralSearch/NeuralSearchSample.cs
Original file line number Diff line number Diff line change
@@ -5,7 +5,6 @@
* compatible open source license.
*/

using System.Diagnostics;
using OpenSearch.Client;
using OpenSearch.Net;

Original file line number Diff line number Diff line change
@@ -46,12 +46,12 @@ internal Indices(ManyIndices indices) : base(indices) { }
internal Indices(IEnumerable<IndexName> indices) : base(new ManyIndices(indices)) { }

/// <summary>All indices. Represents _all</summary>
public static Indices All { get; } = new Indices(new AllIndicesMarker());
public static Indices All { get; } = new(new AllIndicesMarker());

/// <inheritdoc cref="All" />
public static Indices AllIndices { get; } = All;
public static Indices AllIndices => All;

private string DebugDisplay => Match(
private string DebugDisplay => Match(
all => "_all",
types => $"Count: {types.Indices.Count} [" + string.Join(",", types.Indices.Select((t, i) => $"({i + 1}: {t.DebugDisplay})")) + "]"
);
@@ -62,11 +62,13 @@ string IUrlParameter.GetString(IConnectionConfigurationValues settings) => Match
all => "_all",
many =>
{
if (!(settings is IConnectionSettingsValues oscSettings))
throw new Exception(
"Tried to pass index names on querysting but it could not be resolved because no OpenSearch.Client settings are available");
if (settings is not IConnectionSettingsValues oscSettings)
{
throw new Exception(
"Tried to pass index names on querysting but it could not be resolved because no OpenSearch.Client settings are available");
}

var infer = oscSettings.Inferrer;
var infer = oscSettings.Inferrer;
var indices = many.Indices.Select(i => infer.IndexName(i)).Distinct();
return string.Join(",", indices);
}
Loading