From ec2d96d9a814439634966473aefdef08e1bb827c Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Mon, 26 Jun 2023 15:23:25 +1200 Subject: [PATCH] Test against unreleased OpenSearch (#221) Signed-off-by: Thomas Farr --- .github/actions/cached-git-build/action.yml | 48 +++++ .github/workflows/integration.yml | 164 +++++++++++++++--- .../OpenSearch.OpenSearch.Ephemeral.csproj | 1 + .../Plugins/OpenSearchPlugins.cs | 2 + .../DownloadOpenSearchVersion.cs | 11 ++ .../InstallationTasks/InitialConfiguration.cs | 23 ++- .../InitialConfigurationOpenSearch.cs | 148 ---------------- .../Tasks/InstallationTasks/InstallPlugins.cs | 99 +++++++++-- .../packages.lock.json | 6 + .../packages.lock.json | 6 + .../OpenSearch.Stack.ArtifactsApi/Artifact.cs | 26 +-- .../ArtifactBuildState.cs | 37 ---- .../OpenSearchVersion.cs | 29 +--- .../Products/OpenSearchPlugin.cs | 62 +++---- .../Products/Product.cs | 2 - .../Products/SubProduct.cs | 5 - .../Resolvers/ReleasedVersionResolver.cs | 2 +- .../Resolvers/SnapshotApiResolver.cs | 131 -------------- .../Resolvers/StagingVersionResolver.cs | 45 ----- .../packages.lock.json | 6 + .../packages.lock.json | 6 + tests/Tests.Auth.AwsSigV4/packages.lock.json | 6 + .../Tests.ClusterLauncher/packages.lock.json | 6 + .../TestConfigurationBase.cs | 2 - .../Clusters/ReadOnlyCluster.cs | 2 +- .../Clusters/WritableCluster.cs | 20 +-- tests/Tests.Core/packages.lock.json | 6 + tests/Tests.Reproduce/packages.lock.json | 6 + tests/Tests.ScratchPad/packages.lock.json | 6 + .../RootNodeInfo/RootNodeInfoApiTests.cs | 2 +- tests/Tests/packages.lock.json | 6 + 31 files changed, 411 insertions(+), 510 deletions(-) create mode 100644 .github/actions/cached-git-build/action.yml delete mode 100644 abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs delete mode 100644 abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs delete mode 100644 abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs delete mode 100644 abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs diff --git a/.github/actions/cached-git-build/action.yml b/.github/actions/cached-git-build/action.yml new file mode 100644 index 0000000000..b5e427d811 --- /dev/null +++ b/.github/actions/cached-git-build/action.yml @@ -0,0 +1,48 @@ +name: Build and Cache a Git Repository +description: Builds and caches a git repository +inputs: + repository: + description: The git repository to clone + ref: + description: The git ref to clone + path: + description: The path to clone the repository to + cached_paths: + description: A list of paths to cache + build_script: + description: The script to run to build the repository +runs: + using: composite + steps: + - name: Clone Repository + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + path: ${{ inputs.path }} + + - name: Get Commit SHA + id: get-sha + working-directory: ${{ inputs.path }} + shell: bash -eo pipefail {0} + run: echo sha=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + + - name: Restore from cache + id: restore + uses: actions/cache/restore@v3 + with: + path: ${{ inputs.cached_paths }} + key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }} + + - name: Build + if: steps.restore.outputs.cache-hit != 'true' + shell: bash -eo pipefail {0} + working-directory: ${{ inputs.path }} + run: ${{ inputs.build_script }} + + - name: Save to cache + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: ${{ inputs.cached_paths }} + key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e0fce65dcd..2ac79c6a52 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,63 +1,179 @@ name: Integration tests -on: [push, pull_request] +on: + push: + branches-ignore: + - 'dependabot/**' + pull_request: {} + +env: + OPENSEARCH_PLUGINS_DIRECTORY: /tmp/opensearch-plugins jobs: integration-opensearch: name: Integration OpenSearch runs-on: ubuntu-latest - env: - plugins-directory: /tmp/opensearch-plugins strategy: fail-fast: false matrix: # 1.0.1 is skipped because it doesn't have a tag in OpenSearch repo, we can't check out to compile plugins - version: [2.2.0, 2.1.0, 2.0.0, 1.3.1, 1.3.0, 1.2.4, 1.2.3, 1.2.2, 1.2.1, 1.2.0, 1.1.0, 1.0.0] + version: + - 2.7.0 + - 2.6.0 + - 2.5.0 + - 2.4.1 + - 2.3.0 + - 2.2.1 + - 2.1.0 + - 2.0.1 + - 1.3.10 + - 1.2.4 + - 1.1.0 + - 1.0.0 steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + + - uses: actions/setup-dotnet@v1 + with: + dotnet-version: | + 5.0.x + 6.0.x + + - uses: actions/cache@v2 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - run: dotnet nuget locals all --clear + name: Clear nuget cache + working-directory: client + # Due to https://github.com/opensearch-project/project-website/issues/737 # No plugins released for versions 1.0.x , so we need to compile plugins on our own to run tests properly - # Compiled plugins are copied to ${plugins-directory} where picked up by the test framework + # Compiled plugins are copied to ${OPENSEARCH_PLUGINS_DIRECTORY} where picked up by the test framework # Versions of OpenSearch starting 1.1.0 are able to download plugins from the Internet - - name: Clone OpenSearch repository at tag ${{ matrix.version }} - if: ${{ matrix.version == '1.0.0' }} - uses: actions/checkout@v3 + - name: Restore or Build OpenSearch Plugins + uses: ./client/.github/actions/cached-git-build + if: matrix.version == '1.0.0' with: repository: opensearch-project/OpenSearch ref: ${{ matrix.version }} path: opensearch - - name: Make plugins directory - if: ${{ matrix.version == '1.0.0' }} - run: mkdir ${{ env.plugins-directory }} - - name: Build plugins - if: ${{ matrix.version == '1.0.0' }} + cached_paths: | + ./opensearch/plugins/*/build/distributions/*.zip + build_script: | + PluginList=("analysis-icu" "analysis-kuromoji" "analysis-nori" "analysis-phonetic" "ingest-attachment" "mapper-murmur3") + for plugin in ${PluginList[*]}; do + ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=false + done + + - name: Copy OpenSearch Plugins + if: matrix.version == '1.0.0' run: | - cd opensearch - PluginList=("analysis-icu" "analysis-kuromoji" "analysis-nori" "analysis-phonetic" "ingest-attachment" "mapper-murmur3") - for plugin in ${PluginList[*]}; do - ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=false - cp plugins/$plugin/build/distributions/$plugin-${{ matrix.version }}.zip ${{ env.plugins-directory }} - done - - name: Checkout - uses: actions/checkout@v2 + mkdir -p ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }} + cp opensearch/plugins/*/build/distributions/*-${{ matrix.version }}.zip ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }} + + - run: "./build.sh integrate ${{ matrix.version }} readonly,writable random:test_only_one --report" + name: Integration Tests + working-directory: client + + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 + with: + name: report-${{ matrix.version }} + path: client/build/output/* + + integration-opensearch-unreleased: + name: Integration OpenSearch Unreleased + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + opensearch_ref: + - '1.x' + - '2.x' + - 'main' + + steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + - uses: actions/setup-dotnet@v1 with: dotnet-version: | 5.0.x 6.0.x + - uses: actions/cache@v2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} restore-keys: | ${{ runner.os }}-nuget- + - run: dotnet nuget locals all --clear name: Clear nuget cache - - run: "./build.sh integrate ${{ matrix.version }} readonly,writable random:test_only_one --report" + working-directory: client + + - name: Restore or Build OpenSearch + uses: ./client/.github/actions/cached-git-build + with: + repository: opensearch-project/OpenSearch + ref: ${{ matrix.opensearch_ref }} + path: opensearch + cached_paths: | + ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz + ./opensearch/plugins/*/build/distributions/*.zip + build_script: | + ./gradlew :distribution:archives:linux-tar:assemble + + PluginList=("analysis-icu" "analysis-kuromoji" "analysis-nori" "analysis-phonetic" "ingest-attachment" "mapper-murmur3") + for plugin in ${PluginList[*]}; do + ./gradlew :plugins:$plugin:assemble + done + + - name: Restore or Build OpenSearch Security + uses: ./client/.github/actions/cached-git-build + if: matrix.opensearch_ref == '1.x' + with: + repository: opensearch-project/security + ref: ${{ matrix.opensearch_ref }} + path: opensearch-security + cached_paths: | + ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip + build_script: ./gradlew assemble + + - name: Determine OpenSearch distribution path + shell: bash -eo pipefail {0} + run: | + distribution=`ls -1 $PWD/opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz | head -1` + version=`basename $distribution | cut -d'-' -f3,4` + echo "OPENSEARCH_DISTRIBUTION=$distribution" >> $GITHUB_ENV + echo "OPENSEARCH_VERSION=$version" >> $GITHUB_ENV + + mkdir -p $OPENSEARCH_PLUGINS_DIRECTORY + cp ./opensearch/plugins/*/build/distributions/*-$version.zip $OPENSEARCH_PLUGINS_DIRECTORY/ + + if [[ -d "./opensearch-security" ]]; then + cp ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip $OPENSEARCH_PLUGINS_DIRECTORY/ + fi + + - run: "./build.sh integrate ${{ env.OPENSEARCH_VERSION }} readonly,writable random:test_only_one --report" name: Integration Tests + working-directory: client + - name: Upload test report if: failure() uses: actions/upload-artifact@v3 with: - name: report-${{ matrix.version }} - path: build/output/* + name: report-unreleased-${{ matrix.opensearch_ref }} + path: client/build/output/* diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj index 7899b281e4..658fca5d71 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj @@ -9,6 +9,7 @@ true + diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs index c74af2f9eb..7c15f6a5a1 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs @@ -48,6 +48,7 @@ public OpenSearchPlugins(params OpenSearchPlugin[] list) : base(list) { OpenSearchPlugin.AnalysisIcu, OpenSearchPlugin.AnalysisKuromoji, + OpenSearchPlugin.AnalysisNori, OpenSearchPlugin.AnalysisPhonetic, OpenSearchPlugin.AnalysisSmartCn, OpenSearchPlugin.AnalysisStempel, @@ -66,6 +67,7 @@ public OpenSearchPlugins(params OpenSearchPlugin[] list) : base(list) OpenSearchPlugin.RepositoryGCS, OpenSearchPlugin.RepositoryHDFS, OpenSearchPlugin.RepositoryS3, + OpenSearchPlugin.Security, OpenSearchPlugin.StoreSMB, OpenSearchPlugin.DeleteByQuery, }); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs index e652724364..6f2131a7a9 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs @@ -26,6 +26,7 @@ * under the License. */ +using System; using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.Stack.ArtifactsApi.Products; @@ -50,6 +51,16 @@ public override void Run(IEphemeralCluster cluste return; } + if (Environment.GetEnvironmentVariable("OPENSEARCH_DISTRIBUTION") is {} distributionPath) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} copying OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); + File.Copy(distributionPath, to); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} copied OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); + return; + } + cluster.Writer?.WriteDiagnostic( $"{{{nameof(DownloadOpenSearchVersion)}}} downloading OpenSearch [{v}] from {{{from}}} {{{to}}}"); DownloadFile(from, to); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index cc9c204e20..b938890102 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -27,6 +27,7 @@ */ using System.IO; +using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.Stack.ArtifactsApi; @@ -36,25 +37,31 @@ public class InitialConfiguration : ClusterComposeTask { public override void Run(IEphemeralCluster cluster) { - if (cluster.CachingAndCachedHomeExists()) return; - var fs = cluster.FileSystem; - var script = Path.Combine(fs.OpenSearchHome, "server-initial-config.sh"); + var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"); + + if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) + { + cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration"); + return; + } - File.WriteAllText(script, InitialConfigurationOpenSearch.GetConfigurationScript(cluster.ClusterConfiguration.Version)); + var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath); - cluster.Writer?.WriteDiagnostic($"{{{nameof(Run)}}} going to run [server-initial-config.sh]"); + cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, "/bin/bash", - "run initial cluster configuration", - script); + "install security plugin demo configuration", + securityInstallDemoConfig, + "-y", "-i", "-s"); if (cluster.ClusterConfiguration.EnableSsl) return; - File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"), "plugins.security.disabled: true"); + File.AppendAllText(configFile, "plugins.security.disabled: true"); } } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs deleted file mode 100644 index 6dd58b203b..0000000000 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ -/* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -* -* Licensed to Elasticsearch B.V. under one or more contributor -* license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright -* ownership. Elasticsearch B.V. licenses this file to you under -* the Apache License, Version 2.0 (the "License"); you may -* not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -using OpenSearch.Stack.ArtifactsApi; - -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks -{ - internal class InitialConfigurationOpenSearch - { - // Actually, it is content of file `opensearch-tar-install.sh` shipped - // in the tarball, but due to (1) this file might be changed and - // (2) we have to modify the file before execution, because it launches - // the server what we want to do on our own, it is decided to have a - // snapshot of this file. - // The script is taken from v.1.2.4, last 3 lines omitted. - public static string GetConfigurationScript(OpenSearchVersion version) - { - if (version < (OpenSearchVersion)"2.0.0") - return -@"#!/bin/bash - -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 - -OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME -KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/knnlib -##Security Plugin -bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s - -##Perf Plugin -chmod 755 $OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent -chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli -echo ""done security"" -PA_AGENT_JAVA_OPTS=""-Dlog4j.configurationFile=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/log4j2.xml \ - -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ - -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m"" - -OPENSEARCH_MAIN_CLASS=""org.opensearch.performanceanalyzer.PerformanceAnalyzerApp"" \ -OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ -OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS - -if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then - CLK_TCK=`/usr/bin/getconf CLK_TCK` - echo >> $OPENSEARCH_HOME/config/jvm.options - echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options - echo ""-Dclk.tck=$CLK_TCK"" >> $OPENSEARCH_HOME/config/jvm.options - echo ""-Djdk.attach.allowAttachSelf=true"" >> $OPENSEARCH_HOME/config/jvm.options - echo ""-Djava.security.policy=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy"" >> $OPENSEARCH_HOME/config/jvm.options -fi -echo ""done plugins"" - -##Set KNN Dylib Path for macOS and *nix systems -if echo ""$OSTYPE"" | grep -qi ""darwin""; then - if echo ""$JAVA_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then - echo ""k-NN libraries found in JAVA_LIBRARY_PATH"" - else - export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR - echo ""k-NN libraries not found in JAVA_LIBRARY_PATH. Updating path to: $JAVA_LIBRARY_PATH."" - fi -else - if echo ""$LD_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then - echo ""k-NN libraries found in LD_LIBRARY_PATH"" - else - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR - echo ""k-NN libraries not found in LD_LIBRARY_PATH. Updating path to: $LD_LIBRARY_PATH."" - fi -fi -"; - return - //script from 2.0.0 - @"#!/bin/bash - -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 - -export OPENSEARCH_HOME=`dirname $(realpath $0)` -export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config -cd $OPENSEARCH_HOME - -KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib -##Security Plugin -bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s - -echo ""done security"" -PA_AGENT_JAVA_OPTS=""-Dlog4j.configurationFile=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer/log4j2.xml \ - -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ - -XX:MaxRAM=400m"" - -OPENSEARCH_MAIN_CLASS=""org.opensearch.performanceanalyzer.PerformanceAnalyzerApp"" \ -OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ -OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS - -if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_PATH_CONF/jvm.options; then - CLK_TCK=`/usr/bin/getconf CLK_TCK` - echo >> $OPENSEARCH_PATH_CONF/jvm.options - echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_PATH_CONF/jvm.options - echo ""-Dclk.tck=$CLK_TCK"" >> $OPENSEARCH_PATH_CONF/jvm.options - echo ""-Djdk.attach.allowAttachSelf=true"" >> $OPENSEARCH_PATH_CONF/jvm.options - echo ""-Djava.security.policy=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer/opensearch_security.policy"" >> $OPENSEARCH_PATH_CONF/jvm.options - echo ""--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED"" >> $OPENSEARCH_PATH_CONF/jvm.options -fi -echo ""done plugins"" - -##Set KNN Dylib Path for macOS and *nix systems -if echo ""$OSTYPE"" | grep -qi ""darwin""; then - if echo ""$JAVA_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then - echo ""k-NN libraries found in JAVA_LIBRARY_PATH"" - else - export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR - echo ""k-NN libraries not found in JAVA_LIBRARY_PATH. Updating path to: $JAVA_LIBRARY_PATH."" - fi -else - if echo ""$LD_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then - echo ""k-NN libraries found in LD_LIBRARY_PATH"" - else - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR - echo ""k-NN libraries not found in LD_LIBRARY_PATH. Updating path to: $LD_LIBRARY_PATH."" - fi -fi -"; - } - } -} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index bb8e06049e..c64501038e 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -29,17 +29,22 @@ using System; using System.IO; using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using System.Xml; +using JetBrains.Annotations; using OpenSearch.OpenSearch.Managed; using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.OpenSearch.Managed.FileSystem; using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -using ProcNet.Std; namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks { public class InstallPlugins : ClusterComposeTask { + private static readonly HttpClient HttpClient = new(); + public override void Run(IEphemeralCluster cluster) { if (cluster.CachingAndCachedHomeExists()) return; @@ -65,7 +70,7 @@ public override void Run(IEphemeralCluster cluste if (includedByDefault) { cluster.Writer?.WriteDiagnostic( - $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}"); + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}"); continue; } @@ -73,7 +78,7 @@ public override void Run(IEphemeralCluster cluste if (!validForCurrentVersion) { cluster.Writer?.WriteDiagnostic( - $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}"); + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}"); continue; } @@ -81,13 +86,13 @@ public override void Run(IEphemeralCluster cluste if (alreadyInstalled) { cluster.Writer?.WriteDiagnostic( - $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] already installed"); + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] already installed"); continue; } cluster.Writer?.WriteDiagnostic( - $"{{{nameof(Run)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - + $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); + if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); ExecuteBinary( cluster.ClusterConfiguration, @@ -98,17 +103,85 @@ public override void Run(IEphemeralCluster cluste CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); } + + cluster.Writer?.WriteDiagnostic($"{{{nameof(InstallPlugins)}}} all plugins installed"); } private static string GetPluginLocation(OpenSearchPlugin plugin, OpenSearchVersion v) { - // OpenSearch 1.0.0 artifacts were not published. The plugins are built in the workflow and used here. - if (v == "1.0.0") - // The environment variable is set in the integration workflow in - // https://github.com/opensearch-project/opensearch-net/blob/main/.github/workflows/integration.yml - return "file://" + Environment.GetEnvironmentVariable("plugins-directory") + $"/{plugin.SubProductName}-{v}.zip"; - else - return plugin.SubProductName; + var pluginName = plugin.SubProductName; + var versionVariants = new[] + { + v.ToString(), + $"{v.BaseVersion()}.0{(v.IsPreRelease ? $"-{v.PreRelease}" : string.Empty)}", + }; + + if (Environment.GetEnvironmentVariable("OPENSEARCH_PLUGINS_DIRECTORY") is { } pluginsDirectory) + { + foreach (var versionVariant in versionVariants) + { + var pluginFile = Path.Combine(pluginsDirectory, $"{pluginName}-{versionVariant}.zip"); + if (File.Exists(pluginFile)) + { + return new UriBuilder("file",string.Empty) + { + Path = pluginFile + .Replace("%",$"%{(int)'%':X2}") + .Replace("[",$"%{(int)'[':X2}") + .Replace("]",$"%{(int)']':X2}"), + } + .Uri + .AbsoluteUri; + } + } + } + + if (v.IsSnapshot) + return DeterminePluginSnapshotUrl(pluginName, versionVariants); + + return pluginName; + } + + private static string DeterminePluginSnapshotUrl(string pluginName, string[] versionVariants) + { + try + { + var baseUrl = $"https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/{pluginName}"; + + var versionConditions = string.Join(" or ", versionVariants.Select(v => $".='{v}'")); + var version = SelectNodeWithinRemoteXml( + $"{baseUrl}/maven-metadata.xml", + $"metadata/versioning/versions/version[{versionConditions}]") + .InnerText; + + var versionUrl = $"{baseUrl}/{version}"; + + var snapshotVersion = SelectNodeWithinRemoteXml( + $"{versionUrl}/maven-metadata.xml", + "metadata/versioning/snapshotVersions/snapshotVersion[extension='zip']/value") + .InnerText; + + return $"{versionUrl}/{pluginName}-{snapshotVersion}.zip"; + } + catch (Exception e) + { + throw new Exception($"Could not determine snapshot url for plugin `{pluginName}` at versions `{string.Join(", ", versionVariants)}`", e); + } + } + + private static XmlNode SelectNodeWithinRemoteXml(string url, [LanguageInjection("XPath")] string xPath) + { + var task = Task.Run(async () => + { + var msg = await HttpClient.GetAsync(url); + msg.EnsureSuccessStatusCode(); + var xml = await msg.Content.ReadAsStringAsync(); + var doc = new XmlDocument(); + doc.LoadXml(xml); + return doc.SelectSingleNode(xPath) ?? throw new Exception($"Could not find node matching XPath: `{xPath}` within `{xml}`"); + }); + task.Wait(); + return task.Result; } private static void CopyConfigDirectoryToHomeCacheConfigDirectory( diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/packages.lock.json b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/packages.lock.json index f902f47fd4..b631152b32 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/packages.lock.json +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/packages.lock.json @@ -2,6 +2,12 @@ "version": 1, "dependencies": { ".NETStandard,Version=v2.0": { + "JetBrains.Annotations": { + "type": "Direct", + "requested": "[2022.3.1, )", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", "requested": "[1.0.3, )", diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/packages.lock.json b/abstractions/src/OpenSearch.OpenSearch.Xunit/packages.lock.json index 9868b38893..1b25b5c632 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/packages.lock.json +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/packages.lock.json @@ -31,6 +31,11 @@ "xunit.core": "[2.4.2]" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", "resolved": "7.0.0", @@ -830,6 +835,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs index 4d5627aabf..c18e2980e8 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs @@ -38,13 +38,11 @@ public class Artifact { private static readonly Uri BaseUri = new Uri("http://localhost"); - internal Artifact(Product product, Version version, string downloadUrl, ArtifactBuildState state, - string buildHash) + internal Artifact(Product product, Version version, string downloadUrl, string buildHash) { ProductName = product?.ProductName ?? "opensearch"; Version = version; - DownloadUrl = product?.PatchDownloadUrl(downloadUrl); - State = state; + DownloadUrl = downloadUrl; BuildHash = buildHash; } @@ -52,8 +50,7 @@ internal Artifact(Product product, Version version, SearchPackage package, strin { ProductName = product.ProductName; Version = version; - State = ArtifactBuildState.Snapshot; - DownloadUrl = product?.PatchDownloadUrl(package.DownloadUrl); + DownloadUrl = package.DownloadUrl; ShaUrl = package.ShaUrl; AscUrl = package.AscUrl; BuildHash = buildHash; @@ -63,18 +60,10 @@ public string LocalFolderName { get { - var hashed = string.IsNullOrWhiteSpace(BuildHash) ? string.Empty : $"-build-{BuildHash}"; - switch (State) - { - case ArtifactBuildState.Released: - return $"{ProductName}-{Version}"; - case ArtifactBuildState.Snapshot: - return $"{ProductName}-{Version}{hashed}"; - case ArtifactBuildState.BuildCandidate: - return $"{ProductName}-{Version}{hashed}"; - default: - throw new ArgumentOutOfRangeException(nameof(State), $"{State} not expected here"); - } + var hashed = !Version.IsPreRelease || string.IsNullOrWhiteSpace(BuildHash) + ? string.Empty + : $"-build-{BuildHash}"; + return $"{ProductName}-{Version}{hashed}"; } } @@ -96,7 +85,6 @@ public string Archive public Version Version { get; } public string DownloadUrl { get; } - public ArtifactBuildState State { get; } public string BuildHash { get; } public string ShaUrl { get; } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs deleted file mode 100644 index eb0c47d236..0000000000 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ -/* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -* -* Licensed to Elasticsearch B.V. under one or more contributor -* license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright -* ownership. Elasticsearch B.V. licenses this file to you under -* the Apache License, Version 2.0 (the "License"); you may -* not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -namespace OpenSearch.Stack.ArtifactsApi -{ - public enum ArtifactBuildState - { - Released, - Snapshot, - BuildCandidate - } -} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs index 969858173f..c54251faaa 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs @@ -28,7 +28,6 @@ using System; using System.Collections.Concurrent; -using System.Linq; using System.Runtime.InteropServices; using OpenSearch.Stack.ArtifactsApi.Platform; using OpenSearch.Stack.ArtifactsApi.Products; @@ -42,15 +41,12 @@ public class OpenSearchVersion : Version, IComparable { private readonly ConcurrentDictionary _resolved = new(); - protected OpenSearchVersion(string version, ArtifactBuildState state, string buildHash = null) : base(version) - { - ArtifactBuildState = state; - BuildHash = buildHash; - } + protected OpenSearchVersion(string version, string buildHash = null) : base(version) => BuildHash = buildHash; - public ArtifactBuildState ArtifactBuildState { get; } private string BuildHash { get; } + public bool IsSnapshot => PreRelease == "SNAPSHOT"; + public int CompareTo(string other) { var v = (OpenSearchVersion)other; @@ -64,21 +60,8 @@ public Artifact Artifact(Product product) return artifact; var currentPlatform = OsMonikers.CurrentPlatform(); - switch (ArtifactBuildState) - { - case ArtifactBuildState.Released: - ReleasedVersionResolver.TryResolve(product, this, currentPlatform, RuntimeInformation.OSArchitecture, out artifact); - break; - case ArtifactBuildState.Snapshot: - SnapshotApiResolver.TryResolve(product, this, currentPlatform, null, out artifact); - break; - case ArtifactBuildState.BuildCandidate: - StagingVersionResolver.TryResolve(product, this, BuildHash, out artifact); - break; - default: - throw new ArgumentOutOfRangeException(nameof(ArtifactBuildState), - $"{ArtifactBuildState} not expected here"); - } + + ReleasedVersionResolver.TryResolve(product, this, currentPlatform, RuntimeInformation.OSArchitecture, out artifact); _resolved.TryAdd(cacheKey, artifact); @@ -90,7 +73,7 @@ public Artifact Artifact(Product product) /// public static OpenSearchVersion From(string managedVersionString) => // TODO resolve `latest` and `latest-x` for OpenSearch - managedVersionString == null ? null : new OpenSearchVersion(managedVersionString, ArtifactBuildState.Released, ""); + managedVersionString == null ? null : new OpenSearchVersion(managedVersionString, ""); public bool InRange(string range) { diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs index de83502aac..f1ca11056d 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs @@ -35,57 +35,49 @@ public class OpenSearchPlugin : SubProduct { public OpenSearchPlugin(string plugin, Func isValid = null, Func listName = null) - : base(plugin, isValid, listName) - { - PlatformDependent = false; - PatchDownloadUrl = s => - { - //Temporary correct plugin download urls as reported by the snapshot API as it currently has a bug - var correct = $"downloads/opensearch-plugins/{plugin}"; - return !s.Contains(correct) ? s.Replace("downloads/opensearch", correct) : s; - }; - } // ReSharper disable InconsistentNaming - public static OpenSearchPlugin AnalysisIcu { get; } = new OpenSearchPlugin("analysis-icu"); - public static OpenSearchPlugin AnalysisKuromoji { get; } = new OpenSearchPlugin("analysis-kuromoji"); - public static OpenSearchPlugin AnalysisPhonetic { get; } = new OpenSearchPlugin("analysis-phonetic"); - public static OpenSearchPlugin AnalysisSmartCn { get; } = new OpenSearchPlugin("analysis-smartcn"); - public static OpenSearchPlugin AnalysisStempel { get; } = new OpenSearchPlugin("analysis-stempel"); - public static OpenSearchPlugin AnalysisUkrainian { get; } = new OpenSearchPlugin("analysis-ukrainian"); + : base(plugin, isValid, listName) => + PlatformDependent = false; // ReSharper disable InconsistentNaming + public static OpenSearchPlugin AnalysisIcu { get; } = new("analysis-icu"); + public static OpenSearchPlugin AnalysisKuromoji { get; } = new("analysis-kuromoji"); + public static OpenSearchPlugin AnalysisNori { get; } = new("analysis-nori"); + public static OpenSearchPlugin AnalysisPhonetic { get; } = new("analysis-phonetic"); + public static OpenSearchPlugin AnalysisSmartCn { get; } = new("analysis-smartcn"); + public static OpenSearchPlugin AnalysisStempel { get; } = new("analysis-stempel"); + public static OpenSearchPlugin AnalysisUkrainian { get; } = new("analysis-ukrainian"); - public static OpenSearchPlugin DiscoveryAzureClassic { get; } = - new OpenSearchPlugin("discovery-azure-classic"); + public static OpenSearchPlugin DiscoveryAzureClassic { get; } = new("discovery-azure-classic"); - public static OpenSearchPlugin DiscoveryEC2 { get; } = new OpenSearchPlugin("discovery-ec2"); - public static OpenSearchPlugin DiscoveryFile { get; } = new OpenSearchPlugin("discovery-file"); - public static OpenSearchPlugin DiscoveryGCE { get; } = new OpenSearchPlugin("discovery-gce"); + public static OpenSearchPlugin DiscoveryEC2 { get; } = new("discovery-ec2"); + public static OpenSearchPlugin DiscoveryFile { get; } = new("discovery-file"); + public static OpenSearchPlugin DiscoveryGCE { get; } = new("discovery-gce"); - public static OpenSearchPlugin IngestAttachment { get; } = - new OpenSearchPlugin("ingest-attachment", version => version >= "1.0.0"); + public static OpenSearchPlugin IngestAttachment { get; } = new("ingest-attachment", version => version >= "1.0.0"); public static OpenSearchPlugin IngestGeoIp { get; } = - new OpenSearchPlugin("ingest-geoip", version => version >= "1.0.0") + new("ingest-geoip", version => version >= "1.0.0") { ShippedByDefaultAsOf = "1.0.0" }; public static OpenSearchPlugin IngestUserAgent { get; } = - new OpenSearchPlugin("ingest-user-agent", version => version >= "1.0.0") + new("ingest-user-agent", version => version >= "1.0.0") { ShippedByDefaultAsOf = "1.0.0" }; - public static OpenSearchPlugin MapperAttachment { get; } = new OpenSearchPlugin("mapper-attachments"); - public static OpenSearchPlugin MapperMurmur3 { get; } = new OpenSearchPlugin("mapper-murmur3"); - public static OpenSearchPlugin MapperSize { get; } = new OpenSearchPlugin("mapper-size"); + public static OpenSearchPlugin MapperAttachment { get; } = new("mapper-attachments"); + public static OpenSearchPlugin MapperMurmur3 { get; } = new("mapper-murmur3"); + public static OpenSearchPlugin MapperSize { get; } = new("mapper-size"); + + public static OpenSearchPlugin RepositoryAzure { get; } = new("repository-azure"); + public static OpenSearchPlugin RepositoryGCS { get; } = new("repository-gcs"); + public static OpenSearchPlugin RepositoryHDFS { get; } = new("repository-hdfs"); + public static OpenSearchPlugin RepositoryS3 { get; } = new("repository-s3"); - public static OpenSearchPlugin RepositoryAzure { get; } = new OpenSearchPlugin("repository-azure"); - public static OpenSearchPlugin RepositoryGCS { get; } = new OpenSearchPlugin("repository-gcs"); - public static OpenSearchPlugin RepositoryHDFS { get; } = new OpenSearchPlugin("repository-hdfs"); - public static OpenSearchPlugin RepositoryS3 { get; } = new OpenSearchPlugin("repository-s3"); + public static OpenSearchPlugin Security { get; } = new("opensearch-security"); - public static OpenSearchPlugin StoreSMB { get; } = new OpenSearchPlugin("store-smb"); + public static OpenSearchPlugin StoreSMB { get; } = new("store-smb"); - public static OpenSearchPlugin DeleteByQuery { get; } = - new OpenSearchPlugin("delete-by-query", version => version < "1.0.0"); + public static OpenSearchPlugin DeleteByQuery { get; } = new("delete-by-query", version => version < "1.0.0"); } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs index 71e8863e3a..0ab56e4bde 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs @@ -70,7 +70,5 @@ public static Product From(string product, SubProduct subProduct = null, OpenSea public override string ToString() => SubProduct != null ? $"{ProductName}/{SubProduct.SubProductName}" : ProductName; - - public string PatchDownloadUrl(string downloadUrl) => SubProduct?.PatchDownloadUrl(downloadUrl) ?? downloadUrl; } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs index 4d1b6e185f..0f8d713731 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs @@ -48,11 +48,6 @@ public SubProduct(string subProject, Func isValid = nul public OpenSearchVersion ShippedByDefaultAsOf { get; set; } - /// - /// Temporary, snapshot API reports bad plugin download urls - /// - public Func PatchDownloadUrl { get; set; } = s => s; - public bool PlatformDependent { get; protected set; } /// what moniker to use when asserting the sub product is already present diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs index 61f19f03bb..acb2130192 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs @@ -46,7 +46,7 @@ public static bool TryResolve(Product product, Version version, OSPlatform platf var archive = $"{productMoniker}-{version}-{platformMoniker}.{extension}"; var downloadUrl = $"{downloadPath}/{archive}"; - artifact = new Artifact(product, version, downloadUrl, ArtifactBuildState.Released, null); + artifact = new Artifact(product, version, downloadUrl, null); return true; } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs deleted file mode 100644 index 959f1ecc8a..0000000000 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs +++ /dev/null @@ -1,131 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ -/* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -* -* Licensed to Elasticsearch B.V. under one or more contributor -* license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright -* ownership. Elasticsearch B.V. licenses this file to you under -* the Apache License, Version 2.0 (the "License"); you may -* not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading; -using OpenSearch.Stack.ArtifactsApi.Platform; -using OpenSearch.Stack.ArtifactsApi.Products; -using SemanticVersioning; -using Version = SemanticVersioning.Version; - -namespace OpenSearch.Stack.ArtifactsApi.Resolvers -{ - public static class SnapshotApiResolver - { - public static readonly System.Lazy> AvailableVersions = - new System.Lazy>(LoadVersions, LazyThreadSafetyMode.ExecutionAndPublication); - - private static Regex PackageProductRegex { get; } = - new Regex(@"(.*?)-(\d+\.\d+\.\d+(?:-(?:SNAPSHOT|alpha\d+|beta\d+|rc\d+))?)"); - - private static Version IncludeOsMoniker { get; } = new Version("1.0.0"); - - public static Version LatestReleaseOrSnapshot => AvailableVersions.Value.OrderByDescending(v => v).First(); - - public static bool TryResolve(Product product, Version version, OSPlatform os, string filters, - out Artifact artifact) - { - artifact = null; - var p = product.SubProduct?.SubProductName ?? product.ProductName; - var query = p; - if (product.PlatformDependent && version > product.PlatformSuffixAfter) - query += $",{OsMonikers.From(os)}"; - else if (product.PlatformDependent) - query += $",{OsMonikers.CurrentPlatformSearchFilter()}"; - if (!string.IsNullOrWhiteSpace(filters)) - query += $",{filters}"; - - var packages = new Dictionary(); - try - { - var json = ApiResolver.FetchJson($"search/{version}/{query}"); - // if packages is empty it turns into an array[] otherwise its a dictionary :/ - packages = JsonSerializer.Deserialize(json).Packages; - } - catch - { - } - - if (packages == null || packages.Count == 0) return false; - var list = packages - .OrderByDescending(k => k.Value.Classifier == null ? 1 : 0) - .ToArray(); - - var ext = OsMonikers.CurrentPlatformArchiveExtension(); - var shouldEndWith = $"{version}.{ext}"; - if (product.PlatformDependent && version > product.PlatformSuffixAfter) - shouldEndWith = $"{version}-{OsMonikers.CurrentPlatformPackageSuffix()}.{ext}"; - foreach (var kv in list) - { - if (product.PlatformDependent && !kv.Key.EndsWith(shouldEndWith)) continue; - - - var tokens = PackageProductRegex.Split(kv.Key).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); - if (tokens.Length < 2) continue; - - if (!tokens[0].Equals(p, StringComparison.CurrentCultureIgnoreCase)) continue; - if (!tokens[1].Equals(version.ToString(), StringComparison.CurrentCultureIgnoreCase)) continue; - var buildHash = ApiResolver.GetBuildHash(kv.Value.DownloadUrl); - artifact = new Artifact(product, version, kv.Value, buildHash); - } - - return false; - } - - - private static IReadOnlyCollection LoadVersions() - { - var json = ApiResolver.FetchJson("versions"); - var versions = JsonSerializer.Deserialize(json).Versions; - - return new List(versions.Select(v => new Version(v))); - } - - public static Version LatestSnapshotForMajor(int major) - { - var range = new Range($"~{major}"); - return AvailableVersions.Value - .Reverse() - .FirstOrDefault(v => - v.PreRelease == "SNAPSHOT" && range.IsSatisfied(v.ToString().Replace("-SNAPSHOT", ""))); - } - - public static Version LatestReleaseOrSnapshotForMajor(int major) - { - var range = new Range($"~{major}"); - return AvailableVersions.Value - .Reverse() - .FirstOrDefault(v => range.IsSatisfied(v.ToString().Replace("-SNAPSHOT", ""))); - } - } -} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs deleted file mode 100644 index a1191a1b4b..0000000000 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs +++ /dev/null @@ -1,45 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ -/* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -* -* Licensed to Elasticsearch B.V. under one or more contributor -* license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright -* ownership. Elasticsearch B.V. licenses this file to you under -* the Apache License, Version 2.0 (the "License"); you may -* not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -using OpenSearch.Stack.ArtifactsApi.Products; -using Version = SemanticVersioning.Version; - -namespace OpenSearch.Stack.ArtifactsApi.Resolvers -{ - public static class StagingVersionResolver - { - public static bool TryResolve(Product product, Version version, string buildHash, out Artifact artifact) - { - artifact = null; - // Only Linux and only x64 is supported now - var downloadUrl = $"https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/{version}/latest/linux/x64/dist/opensearch/opensearch-{version}-linux-x64.{product.Extension}"; - artifact = new Artifact(product, version, downloadUrl, ArtifactBuildState.BuildCandidate, buildHash); - return true; - } - } -} diff --git a/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/packages.lock.json b/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/packages.lock.json index 31ca48d7c8..257f6c9323 100644 --- a/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/packages.lock.json +++ b/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/packages.lock.json @@ -45,6 +45,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -898,6 +903,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/packages.lock.json b/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/packages.lock.json index 31ca48d7c8..257f6c9323 100644 --- a/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/packages.lock.json +++ b/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/packages.lock.json @@ -45,6 +45,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -898,6 +903,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests.Auth.AwsSigV4/packages.lock.json b/tests/Tests.Auth.AwsSigV4/packages.lock.json index d3066724e2..c1a5e66850 100644 --- a/tests/Tests.Auth.AwsSigV4/packages.lock.json +++ b/tests/Tests.Auth.AwsSigV4/packages.lock.json @@ -50,6 +50,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -910,6 +915,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests.ClusterLauncher/packages.lock.json b/tests/Tests.ClusterLauncher/packages.lock.json index 0fca701a9a..c329a00b94 100644 --- a/tests/Tests.ClusterLauncher/packages.lock.json +++ b/tests/Tests.ClusterLauncher/packages.lock.json @@ -29,6 +29,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -868,6 +873,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests.Configuration/TestConfigurationBase.cs b/tests/Tests.Configuration/TestConfigurationBase.cs index 35a918e793..4f65d00acb 100644 --- a/tests/Tests.Configuration/TestConfigurationBase.cs +++ b/tests/Tests.Configuration/TestConfigurationBase.cs @@ -42,8 +42,6 @@ public abstract class TestConfigurationBase /// The OpenSearch version to test against, defined for both unit and integration tests public OpenSearchVersion OpenSearchVersion { get; protected set; } - public bool OpenSearchVersionIsSnapshot => OpenSearchVersion.ArtifactBuildState == ArtifactBuildState.Snapshot; - /// Force a reseed (bootstrap) of the cluster even if checks indicate bootstrap already ran public bool ForceReseed { get; protected set; } diff --git a/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs b/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs index 4788464d0f..c68c1bd233 100644 --- a/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs +++ b/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs @@ -33,7 +33,7 @@ namespace Tests.Core.ManagedOpenSearch.Clusters { public class ReadOnlyCluster : ClientTestClusterBase { - public ReadOnlyCluster() : base(MapperMurmur3) { } + public ReadOnlyCluster() : base(MapperMurmur3, Security) { } protected override void SeedNode() => new DefaultSeeder(Client).SeedNode(); } diff --git a/tests/Tests.Core/ManagedOpenSearch/Clusters/WritableCluster.cs b/tests/Tests.Core/ManagedOpenSearch/Clusters/WritableCluster.cs index 3083d62e16..56139e5356 100644 --- a/tests/Tests.Core/ManagedOpenSearch/Clusters/WritableCluster.cs +++ b/tests/Tests.Core/ManagedOpenSearch/Clusters/WritableCluster.cs @@ -42,25 +42,15 @@ public class WritableCluster : ClientTestClusterBase { public WritableCluster() : base(CreateConfiguration()) { } - private static ClientTestClusterConfiguration CreateConfiguration() - { - var plugins = new List - { - IngestGeoIp, - IngestAttachment, - AnalysisKuromoji, - AnalysisIcu, - AnalysisPhonetic, + private static ClientTestClusterConfiguration CreateConfiguration() => + new( + AnalysisIcu, AnalysisKuromoji, AnalysisNori, AnalysisPhonetic, + IngestAttachment, IngestGeoIp, MapperMurmur3, - }; - - plugins.Add(new OpenSearchPlugin("analysis-nori")); - - return new ClientTestClusterConfiguration(plugins.ToArray()) + Security) { MaxConcurrency = 4 }; - } protected override void SeedNode() { diff --git a/tests/Tests.Core/packages.lock.json b/tests/Tests.Core/packages.lock.json index 01a2f4048c..781073d887 100644 --- a/tests/Tests.Core/packages.lock.json +++ b/tests/Tests.Core/packages.lock.json @@ -86,6 +86,11 @@ "resolved": "34.0.2", "contentHash": "2KQAuMn3fLAQ2r6jeiffZnpv0bi3GCW3iRB2v1KeHIEBu8MNTh9dBlLTZwGvM0pr+9doKkU8L5JgCURmTmT88A==" }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", "resolved": "7.0.0", @@ -948,6 +953,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests.Reproduce/packages.lock.json b/tests/Tests.Reproduce/packages.lock.json index 31ca48d7c8..257f6c9323 100644 --- a/tests/Tests.Reproduce/packages.lock.json +++ b/tests/Tests.Reproduce/packages.lock.json @@ -45,6 +45,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -898,6 +903,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests.ScratchPad/packages.lock.json b/tests/Tests.ScratchPad/packages.lock.json index aadcb3b6f2..399df4c0d1 100644 --- a/tests/Tests.ScratchPad/packages.lock.json +++ b/tests/Tests.ScratchPad/packages.lock.json @@ -76,6 +76,11 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -1079,6 +1084,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" } diff --git a/tests/Tests/Cluster/RootNodeInfo/RootNodeInfoApiTests.cs b/tests/Tests/Cluster/RootNodeInfo/RootNodeInfoApiTests.cs index b1ad106021..b7e4bca0c0 100644 --- a/tests/Tests/Cluster/RootNodeInfo/RootNodeInfoApiTests.cs +++ b/tests/Tests/Cluster/RootNodeInfo/RootNodeInfoApiTests.cs @@ -64,7 +64,7 @@ protected override void ExpectResponse(RootNodeInfoResponse response) response.Version.BuildDate.Should().BeAfter(default); response.Version.Distribution.Should().NotBeNullOrWhiteSpace(); response.Version.BuildHash.Should().NotBeNullOrWhiteSpace(); - response.Version.BuildSnapshot.Should().Be(TestConfiguration.Instance.OpenSearchVersionIsSnapshot); + response.Version.BuildSnapshot.Should().Be(TestConfiguration.Instance.OpenSearchVersion.IsSnapshot); response.Version.BuildType.Should().NotBeNullOrWhiteSpace(); response.Version.MinimumIndexCompatibilityVersion.Should().NotBeNullOrWhiteSpace(); response.Version.MinimumWireCompatibilityVersion.Should().NotBeNullOrWhiteSpace(); diff --git a/tests/Tests/packages.lock.json b/tests/Tests/packages.lock.json index a14d3be59d..baaeaa7b89 100644 --- a/tests/Tests/packages.lock.json +++ b/tests/Tests/packages.lock.json @@ -72,6 +72,11 @@ "System.Configuration.ConfigurationManager": "4.4.0" } }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2022.3.1", + "contentHash": "11nsS3+lFICGkztVs9PP2NRItxf9fF+qD6xEW/T0YGto52zj07wseUeBdMAU1ah9HNVTDZyRC1u4NWdtJScwhw==" + }, "JunitXml.TestLogger": { "type": "Transitive", "resolved": "3.0.124", @@ -926,6 +931,7 @@ "opensearch.opensearch.ephemeral": { "type": "Project", "dependencies": { + "JetBrains.Annotations": "[2022.3.1, )", "OpenSearch.OpenSearch.Managed": "[1.3.0, )", "SharpZipLib": "[1.4.2, )" }