Skip to content

Commit

Permalink
Run more of the tests that have been ignored or missed (#98)
Browse files Browse the repository at this point in the history 
* dotnet restore

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix Tests.Reproduce dependencies

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Reparent lost Reproduce test

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Move OpenSearch.Stack.ArtifactsApiTests into the correct solution folder

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Run all test projects

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix GithubIssue4197 reproduce

Hard coding dates is not a fun time

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix GithubIssue4243 reproduce

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Run abstractions tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* OpenDistro has been completely archived and downloads no-longer function

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix GithubIssue2306 reproduce

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Remove obsolete ServerType enum

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Make suggested changes

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
  • Loading branch information
@Xtansia
Xtansia committed Oct 14, 2022
1 parent d28d77b commit abd6402
Show file tree
Hide file tree
Showing 65 changed files with 606 additions and 1,499 deletions.
10 changes: 7 additions & 3 deletions OpenSearch.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Net.Auth.AwsSigV4", "src\OpenSearch.Net.Auth.AwsSigV4\OpenSearch.Net.Auth.AwsSigV4.csproj", "{2BDA50C8-767A-4560-9547-15AFC972A6E3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Auth.AwsSigV4", "tests\Tests.Auth.AwsSigV4\Tests.Auth.AwsSigV4.csproj", "{2BCE8150-C16C-4226-B216-C7A0463ADC56}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Stack.ArtifactsApiTests", "abstractions\src\OpenSearch.Stack.ArtifactsApiTests\OpenSearch.Stack.ArtifactsApiTests.csproj", "{1F5A7B1A-2566-481F-91B5-A63D7F939973}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Stack.ArtifactsApiTests", "abstractions\tests\OpenSearch.Stack.ArtifactsApiTests\OpenSearch.Stack.ArtifactsApiTests.csproj", "{1F5A7B1A-2566-481F-91B5-A63D7F939973}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "abstractions", "abstractions", "{87ABA679-F3F4-48CE-82B3-1AAE5D0A5935}"
ProjectSection(SolutionItems) = preProject
Expand All @@ -87,14 +87,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{ABE3B7EE-5B3
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.OpenSearch.Ephemeral", "abstractions\src\OpenSearch.OpenSearch.Ephemeral\OpenSearch.OpenSearch.Ephemeral.csproj", "{31668B33-6157-4A5B-8D4C-18AF760DCA1B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.OpenSearch.EphemeralTests", "abstractions\src\OpenSearch.OpenSearch.EphemeralTests\OpenSearch.OpenSearch.EphemeralTests.csproj", "{C80D225C-F072-4B24-9ACE-82EFD9362237}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.OpenSearch.EphemeralTests", "abstractions\tests\OpenSearch.OpenSearch.EphemeralTests\OpenSearch.OpenSearch.EphemeralTests.csproj", "{C80D225C-F072-4B24-9ACE-82EFD9362237}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.OpenSearch.Managed", "abstractions\src\OpenSearch.OpenSearch.Managed\OpenSearch.OpenSearch.Managed.csproj", "{A9125BAC-71B5-4DC8-879D-1154A0A7DE5C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.OpenSearch.Xunit", "abstractions\src\OpenSearch.OpenSearch.Xunit\OpenSearch.OpenSearch.Xunit.csproj", "{39B7E62B-BDDF-46F2-9306-CE5FC68573A2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Stack.ArtifactsApi", "abstractions\src\OpenSearch.Stack.ArtifactsApi\OpenSearch.Stack.ArtifactsApi.csproj", "{E7C0BDC2-28AD-4582-8FEA-0F6327A42C0E}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{22DF419F-9A90-4317-957D-E239EB3F95DF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -128,10 +130,12 @@ Global
{2BCE8150-C16C-4226-B216-C7A0463ADC56} = {6C4A2627-AF22-4388-9DF7-7A9AEACFD635}
{ABE3B7EE-5B31-4C9A-976A-AD28D257B147} = {87ABA679-F3F4-48CE-82B3-1AAE5D0A5935}
{31668B33-6157-4A5B-8D4C-18AF760DCA1B} = {ABE3B7EE-5B31-4C9A-976A-AD28D257B147}
{C80D225C-F072-4B24-9ACE-82EFD9362237} = {ABE3B7EE-5B31-4C9A-976A-AD28D257B147}
{A9125BAC-71B5-4DC8-879D-1154A0A7DE5C} = {ABE3B7EE-5B31-4C9A-976A-AD28D257B147}
{39B7E62B-BDDF-46F2-9306-CE5FC68573A2} = {ABE3B7EE-5B31-4C9A-976A-AD28D257B147}
{E7C0BDC2-28AD-4582-8FEA-0F6327A42C0E} = {ABE3B7EE-5B31-4C9A-976A-AD28D257B147}
{22DF419F-9A90-4317-957D-E239EB3F95DF} = {87ABA679-F3F4-48CE-82B3-1AAE5D0A5935}
{C80D225C-F072-4B24-9ACE-82EFD9362237} = {22DF419F-9A90-4317-957D-E239EB3F95DF}
{1F5A7B1A-2566-481F-91B5-A63D7F939973} = {22DF419F-9A90-4317-957D-E239EB3F95DF}
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{5B393962-7586-49BA-BD99-3B1E35F48E94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
Expand Down
2 changes: 1 addition & 1 deletion abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ namespace OpenSearch.OpenSearch.Ephemeral
public class EphemeralCluster : EphemeralCluster<EphemeralClusterConfiguration>
{
public EphemeralCluster(OpenSearchVersion version, int numberOfNodes = 1)
: base(new EphemeralClusterConfiguration(version, ServerType.DEFAULT, ClusterFeatures.None, numberOfNodes: numberOfNodes))
: base(new EphemeralClusterConfiguration(version, ClusterFeatures.None, numberOfNodes: numberOfNodes))
{
}

Expand Down
8 changes: 4 additions & 4 deletions abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ namespace OpenSearch.OpenSearch.Ephemeral
{
public class EphemeralClusterConfiguration : ClusterConfiguration<EphemeralFileSystem>
{
public EphemeralClusterConfiguration(OpenSearchVersion version, ServerType serverType, OpenSearchPlugins plugins = null,
public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null,
int numberOfNodes = 1)
: this(version, serverType, ClusterFeatures.None, plugins, numberOfNodes)
: this(version, ClusterFeatures.None, plugins, numberOfNodes)
{
}

public EphemeralClusterConfiguration(OpenSearchVersion version, ServerType serverType, ClusterFeatures features,
public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features,
OpenSearchPlugins plugins = null, int numberOfNodes = 1)
: base(version, serverType, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName)
: base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName)
{
Features = features;

Expand Down
4 changes: 0 additions & 4 deletions abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,6 @@ private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config,
{
{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
// Duplicate all env vars for ES_* prefix for backward compatibility with OpenDistro and ElasticSearch;
// unused env vars would be just ignored.
{config.FileSystem.ConfigEnvironmentVariableName.Replace("OPENSEARCH", "ES"), config.FileSystem.ConfigPath},
{"ES_HOME", config.FileSystem.OpenSearchHome}
}
};

Expand Down
23 changes: 3 additions & 20 deletions ...tions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
* under the License.
*/

using System;
using System.IO;
using OpenSearch.OpenSearch.Managed.ConsoleWriters;
using OpenSearch.Stack.ArtifactsApi;
Expand All @@ -39,19 +38,10 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
{
if (cluster.CachingAndCachedHomeExists()) return;

if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.ElasticSearch)
{
cluster.Writer?.WriteDiagnostic($"{{{nameof(Run)}}} skipping for ElasticSearch");
return;
}

var fs = cluster.FileSystem;
var script = Path.Combine(fs.OpenSearchHome, "server-initial-config.sh");

if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenSearch)
File.WriteAllText(script, InitialConfigurationOpenSearch.GetConfigurationScript(cluster.ClusterConfiguration.Version));
if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro)
File.WriteAllText(script, InitialConfigurationOpenDistro.GetConfigurationScript());
File.WriteAllText(script, InitialConfigurationOpenSearch.GetConfigurationScript(cluster.ClusterConfiguration.Version));

cluster.Writer?.WriteDiagnostic($"{{{nameof(Run)}}} going to run [server-initial-config.sh]");

Expand All @@ -62,16 +52,9 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
"run initial cluster configuration",
script);

if (!cluster.ClusterConfiguration.EnableSsl)
{
if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenSearch)
File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"), "plugins.security.disabled: true");
if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro)
File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "elasticsearch.yml"), "opendistro_security.disabled: true");
}
if (cluster.ClusterConfiguration.EnableSsl) return;

if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.ElasticSearch && cluster.ClusterConfiguration.EnableSsl)
throw new NotImplementedException("ElasticSearch with SSL is not supported");
File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"), "plugins.security.disabled: true");
}
}
}
98 changes: 0 additions & 98 deletions ...OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenDistro.cs
View file

This file was deleted.

5 changes: 0 additions & 5 deletions ...tions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
using System.Net.Http;
using System.Threading;
using OpenSearch.OpenSearch.Managed.ConsoleWriters;
using OpenSearch.Stack.ArtifactsApi;

namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks
{
Expand All @@ -42,10 +41,6 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
void WriteDiagnostic(string message) =>
cluster.Writer?.WriteDiagnostic($"{{{nameof(ValidateRunningVersion)}}} {message}");
var requestedVersion = cluster.ClusterConfiguration.Version;
if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro)
//All supported version of OpenDistro are based on ElasticSearch v.7.10.2
requestedVersion = OpenSearchVersion.From("7.10.2");


WriteDiagnostic($"validating the cluster is running the requested version: {requestedVersion}");

Expand Down
4 changes: 2 additions & 2 deletions abstractions/src/OpenSearch.OpenSearch.Ephemeral/packages.lock.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@
"opensearch.opensearch.managed": {
"type": "Project",
"dependencies": {
"OpenSearch.Stack.ArtifactsApi": "1.0.0",
"OpenSearch.Stack.ArtifactsApi": "1.1.0",
"Proc": "0.6.1",
"System.Net.Http": "4.3.4"
}
Expand Down Expand Up @@ -930,7 +930,7 @@
"opensearch.opensearch.managed": {
"type": "Project",
"dependencies": {
"OpenSearch.Stack.ArtifactsApi": "1.0.0",
"OpenSearch.Stack.ArtifactsApi": "1.1.0",
"Proc": "0.6.1",
"System.Net.Http": "4.3.4"
}
Expand Down
25 changes: 0 additions & 25 deletions ...ions/src/OpenSearch.OpenSearch.EphemeralTests/OpenSearch.OpenSearch.EphemeralTests.csproj
View file

This file was deleted.

2 changes: 1 addition & 1 deletion abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ NodeConfiguration Modify(NodeConfiguration n, int p)
public IDisposable Start(TimeSpan waitForStarted)
{
var nodes = Nodes.Select(n => n.NodeConfiguration.DesiredNodeName).ToArray();
var lineHighlightWriter = new LineHighlightWriter(nodes, LineOutParser.From(ClusterConfiguration.ServerType));
var lineHighlightWriter = new LineHighlightWriter(nodes, LineOutParser.OpenSearch);
return Start(lineHighlightWriter, waitForStarted);
}

Expand Down
33 changes: 15 additions & 18 deletions abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ public interface IClusterConfiguration<out TFileSystem> where TFileSystem : INod
string ClusterName { get; }
NodeSettings DefaultNodeSettings { get; }
OpenSearchVersion Version { get; }
ServerType ServerType { get; }
int NumberOfNodes { get; }
int StartingPortNumber { get; set; }
bool NoCleanupAfterNodeStopped { get; set; }
Expand All @@ -55,17 +54,14 @@ public interface IClusterConfiguration<out TFileSystem> where TFileSystem : INod

public class ClusterConfiguration : ClusterConfiguration<NodeFileSystem>
{
public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, string openSearchHome, int numberOfNodes = 1)
: base(version, serverType, (v, s) => new NodeFileSystem(v, openSearchHome), numberOfNodes, null)
{
}

public ClusterConfiguration(OpenSearchVersion version, ServerType serverType,
Func<OpenSearchVersion, string, NodeFileSystem> fileSystem = null, int numberOfNodes = 1,
string clusterName = null)
: base(version, serverType, fileSystem ?? ((v, s) => new NodeFileSystem(v, s)), numberOfNodes, clusterName)
{
}
public ClusterConfiguration(OpenSearchVersion version, string openSearchHome, int numberOfNodes = 1)
: base(version, (v, s) => new NodeFileSystem(v, openSearchHome), numberOfNodes, null) { }

public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, string, NodeFileSystem> fileSystem = null,
int numberOfNodes = 1,
string clusterName = null
)
: base(version, fileSystem ?? ((v, s) => new NodeFileSystem(v, s)), numberOfNodes, clusterName) { }
}

public class ClusterConfiguration<TFileSystem> : IClusterConfiguration<TFileSystem>
Expand All @@ -81,15 +77,15 @@ public class ClusterConfiguration<TFileSystem> : IClusterConfiguration<TFileSyst
/// </param>
/// <param name="numberOfNodes">The number of nodes in the cluster</param>
/// <param name="clusterName">The name of the cluster</param>
public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, Func<OpenSearchVersion, string, TFileSystem> fileSystem,
int numberOfNodes = 1, string clusterName = null)
public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, string, TFileSystem> fileSystem,
int numberOfNodes = 1, string clusterName = null
)
{
if (fileSystem == null) throw new ArgumentException(nameof(fileSystem));

ClusterName = clusterName;
Version = version;
ServerType = serverType;
Artifact = version.Artifact(Product.From("opensearch", serverType: serverType));
Artifact = version.Artifact(Product.From("opensearch"));
FileSystem = fileSystem(Version, ClusterName);
NumberOfNodes = numberOfNodes;

Expand All @@ -112,7 +108,6 @@ public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, Fu

public string ClusterName { get; }
public OpenSearchVersion Version { get; }
public ServerType ServerType { get; }
public TFileSystem FileSystem { get; }
public int NumberOfNodes { get; }
public int StartingPortNumber { get; set; } = 9200;
Expand All @@ -138,7 +133,7 @@ public virtual string CreateNodeName(int? node) =>
/// <summary>
/// Calculates the quorum given the number of instances
/// </summary>
private static int Quorum(int instanceCount) => Math.Max(1, (int) Math.Floor((double) instanceCount / 2) + 1);
private static int Quorum(int instanceCount) => Math.Max(1, (int)Math.Floor((double)instanceCount / 2) + 1);

/// <summary>
/// Creates a node attribute for the version of OpenSearch
Expand All @@ -155,6 +150,7 @@ public string AttributeKey(string attribute)
protected void Add(string key, string value)
{
if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return;

DefaultNodeSettings.Add(key, value);
}

Expand All @@ -165,6 +161,7 @@ protected void Add(string key, string value)
protected void Add(string key, string value, string range)
{
if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return;

if (string.IsNullOrWhiteSpace(range) || Version.InRange(range))
DefaultNodeSettings.Add(key, value, range);
}
Expand Down
Loading

0 comments on commit abd6402

Please sign in to comment.