Skip to content

Commit

Permalink
feat(backup): extends manifest with info needed for 1-to-1 restore.
Browse files Browse the repository at this point in the history
This adds following data to the backup manifest:
General:
  rack: rack from the scylla configuration
  host_id: host id of the scylla node
Instance Details:
  shard_count: number of shard in scylla node
  storage_size: total size of the disk in bytes
  cloud_provider: aws|gcp|azure or empty in case of on-premise
  instance_type: instance type, e.g. t2.nano or empty when on-premise

Fixes: #4130
  • Loading branch information
VAveryanov8 committed Dec 17, 2024
1 parent a6fecd9 commit f851f3d
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 8 deletions.
25 changes: 19 additions & 6 deletions pkg/service/backup/backupspec/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,25 @@ func (m *ManifestInfo) fileNameParser(v string) error {

// ManifestContent is structure containing information about the backup.
type ManifestContent struct {
Version string `json:"version"`
ClusterName string `json:"cluster_name"`
IP string `json:"ip"`
Size int64 `json:"size"`
Tokens []int64 `json:"tokens"`
Schema string `json:"schema"`
Version string `json:"version"`
ClusterName string `json:"cluster_name"`
IP string `json:"ip"`
Size int64 `json:"size"`
Tokens []int64 `json:"tokens"`
Schema string `json:"schema"`
Rack string `json:"rack"`
HostID string `json:"host_id"`
InstanceDetails InstanceDetails `json:"instance_details"`
}

// InstanceDetails extends backup manifest with additional instance details.
// Mainly needed for 1-to-1 restore.
type InstanceDetails struct {
CloudProvider string `json:"cloud_provider,omitempty"`
InstanceType string `json:"instance_type,omitempty"`

ShardCount int `json:"shard_count"`
StorageSize uint64 `json:"storage_size"`
}

// ManifestContentWithIndex is structure containing information about the backup
Expand Down
56 changes: 54 additions & 2 deletions pkg/service/backup/worker_manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net/http"

"github.com/pkg/errors"
"github.com/scylladb/scylla-manager/v3/pkg/cloudmeta"
"github.com/scylladb/scylla-manager/v3/pkg/scyllaclient"
. "github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/util/parallel"
Expand Down Expand Up @@ -45,11 +46,14 @@ func (w *worker) createAndUploadHostManifest(ctx context.Context, h hostInfo) er
return err
}

m := w.createTemporaryManifest(h, tokens)
m, err := w.createTemporaryManifest(ctx, h, tokens)
if err != nil {
return errors.Wrap(err, "create temp manifest")
}
return w.uploadHostManifest(ctx, h, m)
}

func (w *worker) createTemporaryManifest(h hostInfo, tokens []int64) ManifestInfoWithContent {
func (w *worker) createTemporaryManifest(ctx context.Context, h hostInfo, tokens []int64) (ManifestInfoWithContent, error) {
m := &ManifestInfo{
Location: h.Location,
DC: h.DC,
Expand Down Expand Up @@ -88,10 +92,58 @@ func (w *worker) createTemporaryManifest(h hostInfo, tokens []int64) ManifestInf
c.Size += d.Progress.Size
}

c.HostID = h.ID

rack, err := w.Client.HostRack(ctx, h.IP)
if err != nil {
return ManifestInfoWithContent{}, errors.Wrap(err, "client.HostRack")
}
c.Rack = rack

instanceDetails, err := w.manifestInstanceDetails(ctx, h)
if err != nil {
return ManifestInfoWithContent{}, errors.Wrap(err, "manifest instance details")
}
c.InstanceDetails = instanceDetails

return ManifestInfoWithContent{
ManifestInfo: m,
ManifestContentWithIndex: c,
}, nil
}

// manifestInstanceDetails collects node/instance specific information that's needed for 1-to-1 restore.
func (w *worker) manifestInstanceDetails(ctx context.Context, host hostInfo) (InstanceDetails, error) {
var result InstanceDetails

shardCound, err := w.Client.ShardCount(ctx, host.IP)
if err != nil {
return InstanceDetails{}, errors.Wrap(err, "client.ShardCount")
}
result.ShardCount = int(shardCound)

nodeInfo, err := w.Client.NodeInfo(ctx, host.IP)
if err != nil {
return InstanceDetails{}, errors.Wrap(err, "client.NodeInfo")
}
result.StorageSize = nodeInfo.StorageSize

metaSvc, err := cloudmeta.NewCloudMeta(w.Logger)
if err != nil {
return InstanceDetails{}, errors.Wrap(err, "new cloud meta svc")
}

instanceMeta, err := metaSvc.GetInstanceMetadata(ctx)
if err != nil {
// Metadata may not be available for several reasons:
// 1. running on-premise 2. disabled 3. smth went wrong with metadata server.
// As we cannot distiguish between this cases we can only log err and continue with backup.
w.Logger.Error(ctx, "Get instance metadata", "err", err)
}
result.CloudProvider = string(instanceMeta.CloudProvider)
result.InstanceType = instanceMeta.InstanceType

return result, nil
}

func (w *worker) uploadHostManifest(ctx context.Context, h hostInfo, m ManifestInfoWithContent) error {
Expand Down

0 comments on commit f851f3d

Please sign in to comment.