Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement] Use cluster_snapshot_path to specify the cluster snapshot to be restored #55147

Merged
merged 1 commit into from
Jan 17, 2025
+22 −42
Merged

[Enhancement] Use cluster_snapshot_path to specify the cluster snapshot to be restored #55147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions conf/cluster_snapshot.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

# information about the cluster snapshot to be downloaded and restored
#cluster_snapshot:
# cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
# storage_volume_name: my_s3_volume #defined in storage_volumes
# cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
# cluster_snapshot_name: automated_cluster_snapshot_1704038400000

# do not include leader fe
#frontends:
Expand Down
29 changes: 9 additions & 20 deletions fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,21 @@ public class ClusterSnapshotConfig {
private static final Logger LOG = LogManager.getLogger(ClusterSnapshotConfig.class);

public static class ClusterSnapshot {
@JsonProperty("cluster_snapshot_path")
private String clusterSnapshotPath;

@JsonProperty("storage_volume_name")
private String storageVolumeName;

private StorageVolume storageVolume;

@JsonProperty("cluster_service_id")
private String clusterServiceId;
public String getClusterSnapshotPath() {
return clusterSnapshotPath;
}

@JsonProperty("cluster_snapshot_name")
private String clusterSnapshotName;
public void setClusterSnapshotPath(String clusterSnapshotPath) {
this.clusterSnapshotPath = clusterSnapshotPath;
}

public String getStorageVolumeName() {
return storageVolumeName;
Expand All @@ -68,22 +73,6 @@ public StorageVolume getStorageVolume() {
public void setStorageVolume(StorageVolume storageVolume) {
this.storageVolume = storageVolume;
}

public String getClusterServiceId() {
return clusterServiceId;
}

public void setClusterServiceId(String clusterServiceId) {
this.clusterServiceId = clusterServiceId;
}

public String getClusterSnapshotName() {
return clusterSnapshotName;
}

public void setClusterSnapshotName(String clusterSnapshotName) {
this.clusterSnapshotName = clusterSnapshotName;
}
}

public static class Frontend {
Expand Down
14 changes: 5 additions & 9 deletions fe/fe-core/src/main/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Map;

public class RestoreClusterSnapshotMgr {
private static final Logger LOG = LogManager.getLogger(RestoreClusterSnapshotMgr.class);
Expand Down Expand Up @@ -86,6 +85,7 @@ public static void finishRestoring() throws StarRocksException {
} finally {
self.rollbackConfig();
instance = null;
LOG.info("FE finished to restore from a cluster snapshot");
}
}

Expand Down Expand Up @@ -118,17 +118,13 @@ private void downloadSnapshot() throws StarRocksException {
LOG.info("Deleted image dir {}", localImagePath);
}
if (FileUtils.deleteQuietly(new File(localBdbPath))) {
LOG.info("Deleted bdb {}", localBdbPath);
LOG.info("Deleted bdb dir {}", localBdbPath);
}

ClusterSnapshotConfig.StorageVolume storageVolume = clusterSnapshot.getStorageVolume();
// TODO: use constant and support no snapshot name
String snapshotImagePath = String.join("/", storageVolume.getLocation(), clusterSnapshot.getClusterServiceId(),
"meta/image", clusterSnapshot.getClusterSnapshotName());
Map<String, String> properties = storageVolume.getProperties();
String snapshotImagePath = clusterSnapshot.getClusterSnapshotPath();

LOG.info("Copy snapshot image {} to local dir {}", snapshotImagePath, localImagePath);
HdfsUtil.copyToLocal(snapshotImagePath, localImagePath, properties);
LOG.info("Download cluster snapshot {} to local dir {}", snapshotImagePath, localImagePath);
HdfsUtil.copyToLocal(snapshotImagePath, localImagePath, clusterSnapshot.getStorageVolume().getProperties());
}

private void updateFrontends() throws StarRocksException {
Expand Down
10 changes: 4 additions & 6 deletions fe/fe-core/src/test/java/com/starrocks/lake/snapshot/ClusterSnapshotConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,14 @@ public void testLoadFromFile() {

ClusterSnapshotConfig.ClusterSnapshot clusterSnapshot = config.getClusterSnapshot();
Assert.assertNotNull(clusterSnapshot);
Assert.assertEquals(
"s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000",
clusterSnapshot.getClusterSnapshotPath());
Assert.assertEquals("my_s3_volume", clusterSnapshot.getStorageVolumeName());
Assert.assertNotNull(clusterSnapshot.getStorageVolume());
Assert.assertEquals("f7265e80-631c-44d3-a8ac-cf7cdc7adec811019",
clusterSnapshot.getClusterServiceId());
Assert.assertEquals("automated_cluster_snapshot_1704038400000",
clusterSnapshot.getClusterSnapshotName());

clusterSnapshot.setClusterSnapshotPath(clusterSnapshot.getClusterSnapshotPath());
clusterSnapshot.setStorageVolumeName(clusterSnapshot.getStorageVolumeName());
clusterSnapshot.setClusterServiceId(clusterSnapshot.getClusterServiceId());
clusterSnapshot.setClusterSnapshotName(clusterSnapshot.getClusterSnapshotName());

Assert.assertEquals(2, config.getFrontends().size());
Assert.assertEquals(2, config.getComputeNodes().size());
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgrTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public static void beforeClass() throws Exception {
}

@Test
public void testDownloadSnapshotFaied() throws Exception {
public void testDownloadSnapshotFailed() throws Exception {
Assert.assertThrows(StarRocksException.class, () -> {
RestoreClusterSnapshotMgr.init("src/test/resources/conf/cluster_snapshot.yaml",
new String[] { "-cluster_snapshot" });
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/test/resources/conf/cluster_snapshot.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

# information about the cluster snapshot to be downloaded and restored
cluster_snapshot:
cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
storage_volume_name: my_s3_volume #defined in storage_volumes
cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
cluster_snapshot_name: automated_cluster_snapshot_1704038400000

# do not include leader fe
frontends:
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/test/resources/conf/cluster_snapshot2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

# information about the cluster snapshot to be downloaded and restored
# cluster_snapshot:
# cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
# storage_volume_name: my_s3_volume #defined in storage_volumes
# cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
# cluster_snapshot_name: automated_cluster_snapshot_1704038400000

# do not include leader fe
frontends:
Expand Down
Loading