[
https://issues.apache.org/jira/browse/SOLR-9269?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15388050#comment-15388050
]
ASF GitHub Bot commented on SOLR-9269:
--------------------------------------
Github user hgadre commented on a diff in the pull request:
https://github.com/apache/lucene-solr/pull/52#discussion_r71745628
--- Diff:
solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java
---
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core.snapshots;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This class is responsible to manage the persistent snapshots meta-data
for the Solr indexes. The
+ * persistent snapshots are implemented by relying on Lucene {@linkplain
IndexDeletionPolicy}
+ * abstraction to configure a specific {@linkplain IndexCommit} to be
retained. The
+ * {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to
create/delete the Solr index
+ * snapshots.
+ */
+public class SolrSnapshotMetaDataManager {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata";
+
+ /**
+ * A class defining the meta-data for a specific snapshot.
+ */
+ public static class SnapshotMetaData {
+ private String name;
+ private String indexDirPath;
+ private long generationNumber;
+
+ public SnapshotMetaData(String name, String indexDirPath, long
generationNumber) {
+ super();
+ this.name = name;
+ this.indexDirPath = indexDirPath;
+ this.generationNumber = generationNumber;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getIndexDirPath() {
+ return indexDirPath;
+ }
+
+ public long getGenerationNumber() {
+ return generationNumber;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("SnapshotMetaData[name=");
+ builder.append(name);
+ builder.append(", indexDirPath=");
+ builder.append(indexDirPath);
+ builder.append(", generation=");
+ builder.append(generationNumber);
+ builder.append("]");
+ return builder.toString();
+ }
+ }
+
+ /** Prefix used for the save file. */
+ public static final String SNAPSHOTS_PREFIX = "snapshots_";
+ private static final int VERSION_START = 0;
+ private static final int VERSION_CURRENT = VERSION_START;
+ private static final String CODEC_NAME = "solr-snapshots";
+
+ // The index writer which maintains the snapshots metadata
+ private long nextWriteGen;
+
+ private final Directory dir;
+
+ /** Used to map snapshot name to snapshot meta-data. */
+ protected final Map<String,SnapshotMetaData> nameToDetailsMapping = new
HashMap<>();
+ /** Used to figure out the *current* index data directory path */
+ private final SolrCore solrCore;
+
+ /**
+ * A constructor.
+ *
+ * @param dir The directory where the snapshot meta-data should be
stored. Enables updating
+ * the existing meta-data.
+ * @throws IOException in case of errors.
+ */
+ public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir)
throws IOException {
+ this(solrCore, dir, OpenMode.CREATE_OR_APPEND);
+ }
+
+ /**
+ * A constructor.
+ *
+ * @param dir The directory where the snapshot meta-data is stored.
+ * @param mode CREATE If previous meta-data should be erased.
+ * APPEND If previous meta-data should be read and updated.
+ * CREATE_OR_APPEND Creates a new meta-data structure if one
does not exist
+ * Updates the existing structure if one
exists.
+ * @throws IOException in case of errors.
+ */
+ public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir,
OpenMode mode) throws IOException {
+ this.solrCore = solrCore;
+ this.dir = dir;
+
+ if (mode == OpenMode.CREATE) {
+ deleteSnapshotMetadataFiles();
+ }
+
+ loadFromSnapshotMetadataFile();
+
+ if (mode == OpenMode.APPEND && nextWriteGen == 0) {
+ throw new IllegalStateException("no snapshots stored in this
directory");
+ }
+ }
+
+ /**
+ * @return The snapshot meta-data directory
+ */
+ public Directory getSnapshotsDir() {
+ return dir;
+ }
+
+ /**
+ * This method creates a new snapshot meta-data entry.
+ *
+ * @param name The name of the snapshot.
+ * @param indexDirPath The directory path where the index files are
stored.
+ * @param gen The generation number for the {@linkplain IndexCommit}
being snapshotted.
+ * @throws IOException in case of I/O errors.
+ */
+ public synchronized void snapshot(String name, String indexDirPath, long
gen) throws IOException {
+ Preconditions.checkNotNull(name);
+
+ log.info("Creating the snapshot named {} for core {} associated with
index commit with generation {} in directory {}"
+ , name, solrCore.getName(), gen, indexDirPath);
+
+ if(nameToDetailsMapping.containsKey(name)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name
" + name + " already exists");
+ }
+
+ SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen);
+ nameToDetailsMapping.put(name, d);
+
+ boolean success = false;
+ try {
+ persist();
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ release(name);
+ } catch (Exception e) {
+ // Suppress so we keep throwing original exception
+ }
+ }
+ }
+ }
+
+ /**
+ * This method deletes a previously created snapshot (if any).
+ *
+ * @param name The name of the snapshot to be deleted.
+ * @return The snapshot meta-data if the snapshot with the snapshot name
exists.
+ * @throws IOException in case of I/O error
+ */
+ public synchronized Optional<SnapshotMetaData> release(String name)
throws IOException {
+ log.info("Deleting the snapshot named {} for core {}", name,
solrCore.getName());
+ SnapshotMetaData result =
nameToDetailsMapping.remove(Preconditions.checkNotNull(name));
+ if(result != null) {
+ boolean success = false;
+ try {
+ persist();
+ success = true;
+ } finally {
+ if (!success) {
+ nameToDetailsMapping.put(name, result);
+ }
+ }
+ }
+ return Optional.ofNullable(result);
+ }
+
+ /**
+ * This method returns if snapshot is created for the specified
generation number in
+ * the *current* index directory.
+ *
+ * @param genNumber The generation number for the {@linkplain
IndexCommit} to be checked.
+ * @return true if the snapshot is created.
+ * false otherwise.
+ */
+ public synchronized boolean isSnapshotted(long genNumber) {
+ return isSnapshotted(solrCore.getIndexDir(), genNumber);
+ }
+
+ /**
+ * This method returns if snapshot is created for the specified
generation number in
+ * the specified index directory.
+ *
+ * @param genNumber The generation number for the {@linkplain
IndexCommit} to be checked.
+ * @return true if the snapshot is created.
+ * false otherwise.
+ */
+ public synchronized boolean isSnapshotted(String indexDirPath, long
genNumber) {
+ return nameToDetailsMapping.values().stream()
+ .filter(entry -> entry.getIndexDirPath().equals(indexDirPath) &&
entry.getGenerationNumber() == genNumber)
+ .findFirst()
+ .isPresent();
+ }
+
+ /**
+ * This method returns the snapshot meta-data for the specified name (if
it exists).
+ *
+ * @param name The name of the snapshot
+ * @return The snapshot meta-data if exists.
+ */
+ public synchronized Optional<SnapshotMetaData>
getSnapshotMetaData(String name) {
+ return Optional.ofNullable(nameToDetailsMapping.get(name));
+ }
+
+ /**
+ * @return A list of snapshots created so far.
+ */
+ public synchronized Collection<String> listSnapshots() {
+ return new HashSet<>(nameToDetailsMapping.keySet());
+ }
+
+ /**
+ * This method returns a list of snapshots created in a specified index
directory.
+ *
+ * @param indexDirPath The index directory path.
+ * @return a list snapshots stored in the specified directory.
+ */
+ public synchronized Collection<SnapshotMetaData>
listSnapshotsInIndexDir(String indexDirPath) {
+ return nameToDetailsMapping.entrySet().stream()
+ .filter(entry ->
indexDirPath.equals(entry.getValue().getIndexDirPath()))
+ .map(entry -> entry.getValue())
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * This method returns the {@linkplain IndexCommit} associated with the
specified
+ * <code>commitName</code>. A snapshot with specified
<code>commitName</code> must
+ * be created before invoking this method.
+ *
+ * @param commitName The name of persisted commit
+ * @return the {@linkplain IndexCommit}
+ * @throws IOException in case of I/O error.
+ */
+ public Optional<IndexCommit> getIndexCommitByName(String commitName)
throws IOException {
+ Optional<IndexCommit> result = Optional.empty();
+ Optional<SnapshotMetaData> metaData = getSnapshotMetaData(commitName);
+ if (metaData.isPresent()) {
+ String indexDirPath = metaData.get().getIndexDirPath();
+ long gen = metaData.get().getGenerationNumber();
+
+ Directory d = solrCore.getDirectoryFactory().get(indexDirPath,
DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE);
+ try {
+ result = DirectoryReader.listCommits(d)
+ .stream()
+ .filter(ic -> ic.getGeneration() == gen)
+ .findAny();
+
+ if (!result.isPresent()) {
+ log.warn("Unable to find commit with generation {} in the
directory {}", gen, indexDirPath);
+ }
+
+ } finally {
+ solrCore.getDirectoryFactory().release(d);
+ }
+ } else {
+ log.warn("Commit with name {} is not persisted for core {}",
commitName, solrCore.getName());
+ }
+
+ return result;
+ }
+
+ synchronized private void persist() throws IOException {
--- End diff --
Done.
> Ability to create/delete/list snapshots for a solr core
> -------------------------------------------------------
>
> Key: SOLR-9269
> URL: https://issues.apache.org/jira/browse/SOLR-9269
> Project: Solr
> Issue Type: Sub-task
> Components: SolrCloud
> Reporter: Hrishikesh Gadre
> Assignee: David Smiley
> Attachments: SOLR-9269.patch
>
>
> Support snapshot create/delete/list functionality @ the Solr core level.
> Please refer to parent JIRA for more details.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
