forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
14 changed files
with
309 additions
and
18 deletions.
There are no files selected for viewing
45 changes: 45 additions & 0 deletions
45
server/src/internalClusterTest/java/org/opensearch/remotestore/WarmIndexIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.remotestore; | ||
|
|
||
| import org.opensearch.action.admin.indices.get.GetIndexRequest; | ||
| import org.opensearch.action.admin.indices.get.GetIndexResponse; | ||
| import org.opensearch.cluster.metadata.IndexMetadata; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.indices.replication.common.ReplicationType; | ||
|
|
||
| import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_TYPE_WARM_ENABLED; | ||
| import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; | ||
| import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; | ||
| import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; | ||
| import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; | ||
| import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; | ||
| import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; | ||
|
|
||
| public class WarmIndexIT extends RemoteStoreBaseIntegTestCase { | ||
| public void testWarmIndexCreate() throws Exception { | ||
| Settings settings = Settings.builder() | ||
| .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) | ||
| .put(IndexMetadata.INDEX_TYPE_WARM_ENABLED, true) | ||
| .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) | ||
| .build(); | ||
| assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); | ||
| GetIndexResponse getIndexResponse = client().admin() | ||
| .indices() | ||
| .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) | ||
| .get(); | ||
| Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); | ||
| assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); | ||
| assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); | ||
| assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); | ||
| assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); | ||
| assertEquals("true", indexSettings.get(INDEX_TYPE_WARM_ENABLED)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
server/src/main/java/org/opensearch/index/store/remote/RemoteStoreInterface.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.store.remote; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| public interface RemoteStoreInterface { | ||
| Set<String> getTrackedFiles(); | ||
| FileInfo getFileInfo(String name); | ||
|
|
||
| void delete(String name); | ||
|
|
||
| interface FileInfo { | ||
| long length(); | ||
| } | ||
|
|
||
| } |
139 changes: 139 additions & 0 deletions
139
...er/src/main/java/org/opensearch/index/store/remote/directory/CompositeBlockDirectory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.store.remote.directory; | ||
|
|
||
| import org.apache.lucene.store.AlreadyClosedException; | ||
| import org.apache.lucene.store.Directory; | ||
| import org.apache.lucene.store.FSDirectory; | ||
| import org.apache.lucene.store.FilterDirectory; | ||
| import org.apache.lucene.store.IOContext; | ||
| import org.apache.lucene.store.IndexInput; | ||
| import org.apache.lucene.store.IndexOutput; | ||
| import org.apache.lucene.store.Lock; | ||
| import org.opensearch.index.store.remote.RemoteStoreInterface; | ||
| import org.opensearch.index.store.remote.filecache.FileCache; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Arrays; | ||
| import java.util.Collection; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| public class CompositeBlockDirectory extends Directory { | ||
| private static String BLOCK_EXTENSION = "._block"; | ||
|
|
||
| private FilterDirectory baseDirectory; | ||
| private final FSDirectory fsDirectory; | ||
| private RemoteStoreInterface remote; | ||
|
|
||
| private FileCache cache; | ||
| boolean isOpen; | ||
| public CompositeBlockDirectory(FilterDirectory baseDirectory) { | ||
| this.baseDirectory = baseDirectory; | ||
| this.fsDirectory = (FSDirectory) FilterDirectory.unwrap(baseDirectory); | ||
| isOpen = false; | ||
|
|
||
| } | ||
| public void setRemote(RemoteStoreInterface remoteStore) { | ||
| remote = remoteStore; | ||
| isOpen = false; | ||
| } | ||
|
|
||
| @Override | ||
| protected void ensureOpen() throws AlreadyClosedException { | ||
| // check for isOpen | ||
| } | ||
| private boolean isBlockFile(String file) { | ||
| return file.contains(BLOCK_EXTENSION); | ||
| } | ||
|
|
||
| @Override | ||
| public String[] listAll() throws IOException { | ||
| Set<String> allFiles = Arrays.asList(baseDirectory.listAll()).stream().filter(file -> isBlockFile(file) == false).collect(Collectors.toSet()); | ||
| allFiles.addAll(remote.getTrackedFiles()); | ||
|
|
||
| String[] files = new String[allFiles.size()]; | ||
| allFiles.toArray(files); | ||
| Arrays.sort(files); | ||
|
|
||
| return files; | ||
| } | ||
|
|
||
| @Override | ||
| public void deleteFile(String name) throws IOException { | ||
| // TODO: Add support for pending deletions. | ||
| // TODO: Add support for deleting block files? needed? | ||
|
|
||
| if(remote.getTrackedFiles().contains(name)) { | ||
| remote.delete(name); | ||
| } | ||
| // assuming its tracked in cache. | ||
| if (!cache.remove(fsDirectory.getDirectory().resolve(name))) { | ||
| baseDirectory.deleteFile(name); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public long fileLength(String name) throws IOException { | ||
| if(remote.getTrackedFiles().contains(name)) { | ||
| return remote.getFileInfo(name).length(); | ||
| } else { | ||
| return baseDirectory.fileLength(name); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public IndexOutput createOutput(String name, IOContext context) throws IOException { | ||
| return baseDirectory.createOutput(name, context); | ||
| } | ||
|
|
||
| @Override | ||
| public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException { | ||
| return baseDirectory.createTempOutput(prefix, suffix, context); | ||
| } | ||
|
|
||
| @Override | ||
| public void sync(Collection<String> names) throws IOException { | ||
| // TODO: what sync means for composite directory. | ||
| Set<String> remoteFiles = remote.getTrackedFiles(); | ||
| baseDirectory.sync(names.stream().filter(remoteFiles::contains).collect(Collectors.toSet())); | ||
| } | ||
|
|
||
| @Override | ||
| public void syncMetaData() throws IOException { | ||
| // TODO: what sync metadata means for composite directory. | ||
| } | ||
|
|
||
| @Override | ||
| public void rename(String source, String dest) throws IOException { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public IndexInput openInput(String name, IOContext context) throws IOException { | ||
| // TODO: | ||
| return baseDirectory.openInput(name, context); | ||
| } | ||
|
|
||
| @Override | ||
| public Lock obtainLock(String name) throws IOException { | ||
| // TODO: Lock implications for composite dir, | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public Set<String> getPendingDeletions() throws IOException { | ||
| return null; // pending deletions integration pending. | ||
| } | ||
| } |
Oops, something went wrong.