Adding ability to override directory permissions for db backup (#8888)

* Adding ability to override directory permissions for db backup

* adding new flag to usage.go

* Backup now has its own directory handling method to reduce footprint of prospective change

* removing unneeded test

* fixing build error

* switching to url param for override

* stripping flag

* adding in unit tests

* fixing a test

* minor changes for testing

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

beacon-chain/db/kafka/passthrough.go

@@ -25,8 +25,8 @@ func (e Exporter) ClearDB() error {
 }
 
 // Backup -- passthrough.
-func (e Exporter) Backup(ctx context.Context, outputDir string) error {
+func (e Exporter) Backup(ctx context.Context, outputDir string, overridePermission bool) error {
-	return e.db.Backup(ctx, outputDir)
+	return e.db.Backup(ctx, outputDir, false)
 }
 
 // Block -- passthrough.

beacon-chain/db/kv/backup.go

@@ -16,7 +16,7 @@ const backupsDirectoryName = "backups"
 
 // Backup the database to the datadir backup directory.
 // Example for backup at slot 345: $DATADIR/backups/prysm_beacondb_at_slot_0000345.backup
-func (s *Store) Backup(ctx context.Context, outputDir string) error {
+func (s *Store) Backup(ctx context.Context, outputDir string, permissionOverride bool) error {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.Backup")
 	defer span.End()
 
@@ -38,7 +38,7 @@ func (s *Store) Backup(ctx context.Context, outputDir string) error {
 		return errors.New("no head block")
 	}
 	// Ensure the backups directory exists.
-	if err := fileutil.MkdirAll(backupsDir); err != nil {
+	if err := fileutil.HandleBackupDir(backupsDir, permissionOverride); err != nil {
 		return err
 	}
 	backupPath := path.Join(backupsDir, fmt.Sprintf("prysm_beacondb_at_slot_%07d.backup", head.Block().Slot()))

beacon-chain/db/kv/backup_test.go

@@ -29,7 +29,7 @@ func TestStore_Backup(t *testing.T) {
 	require.NoError(t, db.SaveState(ctx, st, root))
 	require.NoError(t, db.SaveHeadBlockRoot(ctx, root))
 
-	require.NoError(t, db.Backup(ctx, ""))
+	require.NoError(t, db.Backup(ctx, "", false))
 
 	backupsPath := filepath.Join(db.databasePath, backupsDirectoryName)
 	files, err := ioutil.ReadDir(backupsPath)
@@ -71,7 +71,7 @@ func TestStore_BackupMultipleBuckets(t *testing.T) {
 		require.NoError(t, db.SaveHeadBlockRoot(ctx, root))
 	}
 
-	require.NoError(t, db.Backup(ctx, ""))
+	require.NoError(t, db.Backup(ctx, "", false))
 
 	backupsPath := filepath.Join(db.databasePath, backupsDirectoryName)
 	files, err := ioutil.ReadDir(backupsPath)

shared/backuputil/http_backup_handler.go

@@ -10,17 +10,19 @@ import (
 
 // BackupExporter defines a backup exporter methods.
 type BackupExporter interface {
-	Backup(ctx context.Context, outputPath string) error
+	Backup(ctx context.Context, outputPath string, permissionOverride bool) error
 }
 
 // BackupHandler for accepting requests to initiate a new database backup.
 func BackupHandler(bk BackupExporter, outputDir string) func(http.ResponseWriter, *http.Request) {
 	log := logrus.WithField("prefix", "db")
 
-	return func(w http.ResponseWriter, _ *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		log.Debug("Creating database backup from HTTP webhook")
 
-		if err := bk.Backup(context.Background(), outputDir); err != nil {
+		_, permissionOverride := r.URL.Query()["permissionOverride"]
+
+		if err := bk.Backup(context.Background(), outputDir, permissionOverride); err != nil {
 			log.WithError(err).Error("Failed to create backup")
 			w.WriteHeader(http.StatusInternalServerError)
 			return

shared/fileutil/fileutil.go

@@ -32,6 +32,34 @@ func ExpandPath(p string) (string, error) {
 	return filepath.Abs(path.Clean(os.ExpandEnv(p)))
 }
 
+// HandleBackupDir takes an input directory path and either alters its permissions to be usable if it already exists, creates it if not
+func HandleBackupDir(dirPath string, permissionOverride bool) error {
+	expanded, err := ExpandPath(dirPath)
+	if err != nil {
+		return err
+	}
+	exists, err := HasDir(expanded)
+	if err != nil {
+		return err
+	}
+	if exists {
+		info, err := os.Stat(expanded)
+		if err != nil {
+			return err
+		}
+		if info.Mode().Perm() != params.BeaconIoConfig().ReadWriteExecutePermissions {
+			if permissionOverride {
+				if err := os.Chmod(expanded, params.BeaconIoConfig().ReadWriteExecutePermissions); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("dir already exists without proper 0700 permissions")
+			}
+		}
+	}
+	return os.MkdirAll(expanded, params.BeaconIoConfig().ReadWriteExecutePermissions)
+}
+
 // MkdirAll takes in a path, expands it if necessary, and looks through the
 // permissions of every directory along the path, ensuring we are not attempting
 // to overwrite any existing permissions. Finally, creates the directory accordingly

shared/fileutil/fileutil_test.go

@@ -56,13 +56,48 @@ func TestMkdirAll_AlreadyExists_WrongPermissions(t *testing.T) {
 	assert.ErrorContains(t, "already exists without proper 0700 permissions", err)
 }
 
-func TestMkdirAll_AlreadyExists_OK(t *testing.T) {
+func TestMkdirAll_AlreadyExists_Override(t *testing.T) {
 	dirName := t.TempDir() + "somedir"
 	err := os.MkdirAll(dirName, params.BeaconIoConfig().ReadWriteExecutePermissions)
 	require.NoError(t, err)
 	assert.NoError(t, fileutil.MkdirAll(dirName))
 }
 
+func TestHandleBackupDir_AlreadyExists_Override(t *testing.T) {
+	dirName := t.TempDir() + "somedir"
+	err := os.MkdirAll(dirName, os.ModePerm)
+	require.NoError(t, err)
+	info, err := os.Stat(dirName)
+	require.NoError(t, err)
+	assert.Equal(t, "drwxr-xr-x", info.Mode().String())
+	assert.NoError(t, fileutil.HandleBackupDir(dirName, true))
+	info, err = os.Stat(dirName)
+	require.NoError(t, err)
+	assert.Equal(t, "drwx------", info.Mode().String())
+}
+
+func TestHandleBackupDir_AlreadyExists_No_Override(t *testing.T) {
+	dirName := t.TempDir() + "somedir"
+	err := os.MkdirAll(dirName, os.ModePerm)
+	require.NoError(t, err)
+	info, err := os.Stat(dirName)
+	require.NoError(t, err)
+	assert.Equal(t, "drwxr-xr-x", info.Mode().String())
+	err = fileutil.HandleBackupDir(dirName, false)
+	assert.ErrorContains(t, "dir already exists without proper 0700 permissions", err)
+	info, err = os.Stat(dirName)
+	require.NoError(t, err)
+	assert.Equal(t, "drwxr-xr-x", info.Mode().String())
+}
+
+func TestHandleBackupDir_NewDir(t *testing.T) {
+	dirName := t.TempDir() + "somedir"
+	require.NoError(t, fileutil.HandleBackupDir(dirName, true))
+	info, err := os.Stat(dirName)
+	require.NoError(t, err)
+	assert.Equal(t, "drwx------", info.Mode().String())
+}
+
 func TestMkdirAll_OK(t *testing.T) {
 	dirName := t.TempDir() + "somedir"
 	err := fileutil.MkdirAll(dirName)

slasher/db/kv/backup.go

@@ -16,7 +16,7 @@ const backupsDirectoryName = "backups"
 
 // Backup the database to the datadir backup directory.
 // Example for backup: $DATADIR/backups/prysm_slasherdb_10291092.backup
-func (s *Store) Backup(ctx context.Context, outputDir string) error {
+func (s *Store) Backup(ctx context.Context, outputDir string, overridePermission bool) error {
 	ctx, span := trace.StartSpan(ctx, "SlasherDB.Backup")
 	defer span.End()
 
@@ -31,7 +31,7 @@ func (s *Store) Backup(ctx context.Context, outputDir string) error {
 		backupsDir = path.Join(s.databasePath, backupsDirectoryName)
 	}
 	// Ensure the backups directory exists.
-	if err := fileutil.MkdirAll(backupsDir); err != nil {
+	if err := fileutil.HandleBackupDir(backupsDir, overridePermission); err != nil {
 		return err
 	}
 	backupPath := path.Join(backupsDir, fmt.Sprintf("prysm_slasherdb_%d.backup", time.Now().Unix()))

slasher/db/kv/backup_test.go

@@ -16,7 +16,7 @@ func TestStore_Backup(t *testing.T) {
 	ctx := context.Background()
 	pubKey := []byte("hello")
 	require.NoError(t, db.SavePubKey(ctx, types.ValidatorIndex(1), pubKey))
-	require.NoError(t, db.Backup(ctx, ""))
+	require.NoError(t, db.Backup(ctx, "", false))
 
 	backupsPath := filepath.Join(db.databasePath, backupsDirectoryName)
 	files, err := ioutil.ReadDir(backupsPath)

validator/db/kv/backup.go

@@ -16,7 +16,7 @@ const backupsDirectoryName = "backups"
 
 // Backup the database to the datadir backup directory.
 // Example for backup: $DATADIR/backups/prysm_validatordb_1029019.backup
-func (s *Store) Backup(ctx context.Context, outputDir string) error {
+func (s *Store) Backup(ctx context.Context, outputDir string, permissionOverride bool) error {
 	ctx, span := trace.StartSpan(ctx, "ValidatorDB.Backup")
 	defer span.End()
 
@@ -31,7 +31,7 @@ func (s *Store) Backup(ctx context.Context, outputDir string) error {
 		backupsDir = path.Join(s.databasePath, backupsDirectoryName)
 	}
 	// Ensure the backups directory exists.
-	if err := fileutil.MkdirAll(backupsDir); err != nil {
+	if err := fileutil.HandleBackupDir(backupsDir, permissionOverride); err != nil {
 		return err
 	}
 	backupPath := path.Join(backupsDir, fmt.Sprintf("prysm_validatordb_%d.backup", time.Now().Unix()))

validator/db/kv/backup_test.go

@@ -15,7 +15,7 @@ func TestStore_Backup(t *testing.T) {
 	ctx := context.Background()
 	root := [32]byte{1}
 	require.NoError(t, db.SaveGenesisValidatorsRoot(ctx, root[:]))
-	require.NoError(t, db.Backup(ctx, ""))
+	require.NoError(t, db.Backup(ctx, "", true))
 
 	backupsPath := filepath.Join(db.databasePath, backupsDirectoryName)
 	files, err := ioutil.ReadDir(backupsPath)