Add migration code for BoltDB to SQLite

This is gated behind a new option in `podman system migrate`,
`--migrate-db`, or by a system restart being performed.

BoltDB support was removed in Podman 6, so we are certain that,
when we start Podman, a SQLite state is in use. However, if we
also detect a valid BoltDB state, we will attempt a migration.

Migration is performed by retrieving all volumes, pods, and
containers (in that order, to ensure there are no dependency
conflicts) from the Bolt database, when adding them to the SQLite
database. If there is a conflict - IE, a container exists in both
SQLite and Bolt - we skip migration for that object. The old DB
is then renamed so we do not try to migrate it again.

Our ability to test complex migration scenarios is limited, but
this should handle simple migrations easily.

This is a heavily adapted version of #27660 rebuilt to work with
Podman 6.0.

This cannot be tested automatically, as the ability to create Bolt
databases has been entirely removed with Podman 6.

Signed-off-by: Matt Heon <matthew.heon@pm.me>

Author: mheon

cmd/podman/system/migrate.go

@@ -44,6 +44,8 @@ func init() {
 	newRuntimeFlagName := "new-runtime"
 	flags.StringVar(&migrateOptions.NewRuntime, newRuntimeFlagName, "", "Specify a new runtime for all containers")
 	_ = migrateCommand.RegisterFlagCompletionFunc(newRuntimeFlagName, completion.AutocompleteNone)
+
+	flags.BoolVar(&migrateOptions.MigrateDB, "migrate-db", false, "Migrate database from BoltDB to SQLite")
 }
 
 func migrate(_ *cobra.Command, _ []string) error {

docs/source/markdown/podman-system-migrate.1.md

@@ -26,6 +26,15 @@ newly configured mappings.
 
 ## OPTIONS
 
+#### **--migrate-db**
+
+Migrate from the legacy BoltDB database to SQLite.
+Support for BoltDB will be removed in Podman 6.0.
+Podman will display a warning if this migration is necessary.
+To ensure complete migration, all other Podman commands should be shut down before database migration.
+In particular, systemd-activated services like **podman system service** and Quadlets should be manually stopped prior to migration.
+The legacy database will not be removed, so no data loss should occur even on failure.
+
 #### **--new-runtime**=*runtime*
 
 Set a new OCI runtime for all containers.

libpod/container_graph.go

@@ -289,6 +289,89 @@ func startNode(ctx context.Context, node *containerNode, setError bool, ctrError
 	}
 }
 
+// Migrates all nodes to the new SQLite state
+func migrateNodeDatabase(node *containerNode, setError bool, ctrErrors map[string]error, ctrsVisited map[string]bool, sqliteState State) {
+	// First, check if we have already visited the node
+	if ctrsVisited[node.id] {
+		return
+	}
+
+	// If setError is true, a dependency of us failed
+	// Mark us as failed and recurse
+	if setError {
+		// Mark us as visited, and set an error
+		ctrsVisited[node.id] = true
+		ctrErrors[node.id] = fmt.Errorf("a dependency of container %s failed to migrate: %w", node.id, define.ErrCtrStateInvalid)
+
+		// Hit anyone who depends on us, and set errors on them too
+		for _, successor := range node.dependedOn {
+			migrateNodeDatabase(successor, true, ctrErrors, ctrsVisited, sqliteState)
+		}
+
+		return
+	}
+
+	// Have all our dependencies started?
+	// If not, don't visit the node yet
+	depsVisited := true
+	for _, dep := range node.dependsOn {
+		depsVisited = depsVisited && ctrsVisited[dep.id]
+	}
+	if !depsVisited {
+		// Don't visit us yet, all dependencies are not up
+		// We'll hit the dependencies eventually, and when we do it will
+		// recurse here
+		return
+	}
+
+	// Going to try to migrate the container, mark us as visited
+	ctrsVisited[node.id] = true
+
+	ctrErrored := false
+	ctrExists := false
+
+	// Add and save the container
+	if node.container.config.Pod == "" {
+		if err := sqliteState.AddContainer(node.container); err != nil {
+			if errors.Is(err, define.ErrPodExists) {
+				logrus.Warnf("Container with name %s already exists in the SQLite database; refusing to migrate from BoltDB", node.container.Name())
+				ctrExists = true
+			} else {
+				ctrErrored = true
+				ctrErrors[node.id] = err
+			}
+		}
+	} else {
+		// We don't actually *use* the pod in this operation, other than its ID...
+		// So fake it
+		pod := new(Pod)
+		pod.config = new(PodConfig)
+		pod.config.ID = node.container.config.Pod
+		pod.valid = true
+
+		if err := sqliteState.AddContainer(node.container); err != nil {
+			if errors.Is(err, define.ErrPodExists) {
+				logrus.Warnf("Container with name %s already exists in the SQLite database; refusing to migrate from BoltDB", node.container.Name())
+				ctrExists = true
+			} else {
+				ctrErrored = true
+				ctrErrors[node.id] = err
+			}
+		}
+	}
+	if !ctrErrored && !ctrExists {
+		if err := sqliteState.SaveContainer(node.container); err != nil {
+			ctrErrored = true
+			ctrErrors[node.id] = err
+		}
+	}
+
+	// Recurse to anyone who depends on us and start them
+	for _, successor := range node.dependedOn {
+		migrateNodeDatabase(successor, ctrErrored, ctrErrors, ctrsVisited, sqliteState)
+	}
+}
+
 // Contains all details required for traversing the container graph.
 type nodeTraversal struct {
 	// Optional. but *MUST* be locked.

libpod/runtime.go

@@ -792,6 +792,16 @@ func (r *Runtime) Shutdown(force bool) error {
 func (r *Runtime) refresh(ctx context.Context, alivePath string) error {
 	logrus.Debugf("Podman detected system restart - performing state refresh")
 
+	if err := r.checkCanMigrate(); err != nil {
+		if errors.Is(err, errCannotMigrateHardcodedBolt) {
+			logrus.Infof("Refusing to automatically migrate from BoltDB to SQLite as BoltDB is hardcoded in containers.conf")
+		}
+	} else {
+		if err := r.migrateDB(); err != nil {
+			logrus.Errorf("Automatic migration from BoltDB to SQLite failed: %v", err)
+		}
+	}
+
 	// Clear state of database if not running in container
 	if !graphRootMounted() {
 		// First clear the state in the database

libpod/runtime_migrate.go

@@ -3,19 +3,23 @@
 package libpod
 
 import (
+	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/sirupsen/logrus"
+	"go.podman.io/common/pkg/config"
 	"go.podman.io/podman/v6/libpod/define"
 	"go.podman.io/podman/v6/pkg/namespaces"
+	"go.podman.io/storage/pkg/fileutils"
 )
 
 // Migrate stops the rootless pause process and performs any necessary database
 // migrations that are required. It can also migrate all containers to a new OCI
 // runtime, if requested.
-func (r *Runtime) Migrate(newRuntime string) error {
+func (r *Runtime) Migrate(newRuntime string, migrateDB bool) error {
 	// Acquire the alive lock and hold it.
 	// Ensures that we don't let other Podman commands run while we are
 	// rewriting things in the DB.
@@ -97,5 +101,161 @@ func (r *Runtime) Migrate(newRuntime string) error {
 		}
 	}
 
+	if migrateDB {
+		if err := r.checkCanMigrate(); err != nil {
+			switch {
+			case errors.Is(err, errCannotMigrateNoBolt):
+				fmt.Printf("No migration is necessary: %v", err)
+				return r.stopPauseProcess()
+			case errors.Is(err, errCannotMigrateHardcodedBolt):
+				logrus.Errorf("In containers.conf, database_backend is manually set to \"boltdb\" - comment this line out and run `podman system migrate --migrate-db` or restart the system to complete migration to SQLite")
+				return fmt.Errorf("unable to migrate to SQLite database as database backend manually set")
+			default:
+				return err
+			}
+		}
+
+		if err := r.migrateDB(); err != nil {
+			return fmt.Errorf("migrating database from BoltDB to SQLite: %w", err)
+		}
+	}
+
 	return r.stopPauseProcess()
 }
+
+var (
+	errCannotMigrateNoBolt        = errors.New("no BoltDB database to migrate")
+	errCannotMigrateHardcodedBolt = errors.New("database_backend in containers.conf is manually set to \"boltdb\"")
+)
+
+func (r *Runtime) checkCanMigrate() error {
+	boltPath := getBoltDBPath(r)
+	if err := fileutils.Exists(boltPath); err != nil {
+		return errCannotMigrateNoBolt
+	}
+
+	// Necessary as database configuration is overwritten when the state is set up.
+	// So we need a completely new state from disk to see what the user set.
+	newCfg, err := config.New(nil)
+	if err != nil {
+		return fmt.Errorf("reloading configuration to check database backend in use")
+	}
+	backend, err := config.ParseDBBackend(newCfg.Engine.DBBackend)
+	if err != nil {
+		return fmt.Errorf("invalid DB backend configured - please change containers.conf database_backend to \"sqlite\"")
+	}
+	if backend == config.DBBackendBoltDB {
+		return errCannotMigrateHardcodedBolt
+	}
+
+	return nil
+}
+
+func (r *Runtime) migrateDB() error {
+	boltPath := getBoltDBPath(r)
+	// Get us a Bolt database
+	oldState, err := NewBoltState(boltPath, r)
+	if err != nil {
+		return fmt.Errorf("opening legacy Bolt database at %s: %w", boltPath, err)
+	}
+
+	// Migrate volumes, then pods, then containers.
+	// Containers must be last as the pods they are part of and volumes they use must already exist.
+	allVolumes, err := oldState.AllVolumes()
+	if err != nil {
+		return fmt.Errorf("retrieving volumes from boltdb: %w", err)
+	}
+	for _, vol := range allVolumes {
+		if err := r.state.AddVolume(vol); err != nil {
+			if errors.Is(err, define.ErrVolumeExists) {
+				logrus.Warnf("Volume with name %s already exists in the SQLite database; refusing to migrate from BoltDB", vol.Name())
+				continue
+			}
+			return err
+		}
+		if err := oldState.UpdateVolume(vol); err != nil {
+			return err
+		}
+		if err := r.state.SaveVolume(vol); err != nil {
+			return err
+		}
+	}
+
+	allPods, err := oldState.AllPods()
+	if err != nil {
+		return fmt.Errorf("retrieving pods from boltdb: %w", err)
+	}
+	for _, pod := range allPods {
+		if err := r.state.AddPod(pod); err != nil {
+			if errors.Is(err, define.ErrPodExists) {
+				logrus.Warnf("Pod with name %s already exists in the SQLite database; refusing to migrate from BoltDB", pod.Name())
+				continue
+			}
+			return err
+		}
+		if err := oldState.UpdatePod(pod); err != nil {
+			return err
+		}
+		if err := r.state.SavePod(pod); err != nil {
+			return err
+		}
+	}
+
+	// Containers must be done as a graph due to dependencies.
+	// The state will error if we add a container before its dependencies.
+	allCtrs, err := oldState.AllContainers(true)
+	if err != nil {
+		return fmt.Errorf("retrieving containers from boltdb: %w", err)
+	}
+
+	// BoltDB doesn't actually populate container networks on initial pull
+	// from the database, that needs to be done separately.
+	for _, ctr := range allCtrs {
+		ctrNetworks, err := oldState.GetNetworks(ctr)
+		if err != nil {
+			return err
+		}
+		ctr.config.Networks = convertLegacyNetworks(ctrNetworks)
+	}
+
+	graph, err := BuildContainerGraph(allCtrs)
+	if err != nil {
+		return err
+	}
+
+	ctrErrors := make(map[string]error)
+	ctrsVisited := make(map[string]bool)
+
+	for _, node := range graph.noDepNodes {
+		migrateNodeDatabase(node, false, ctrErrors, ctrsVisited, r.state)
+	}
+	var ctrError error
+	for id, err := range ctrErrors {
+		if ctrError != nil {
+			logrus.Errorf("Migrating containers to SQLite: %v", ctrError)
+		}
+		ctrError = fmt.Errorf("migrating container %s: %w", id, err)
+	}
+	if ctrError != nil {
+		return ctrError
+	}
+
+	oldState.Close()
+
+	// Move the Bolt database so it is not reused, but preserve so data is not lost.
+	newBoltDBPath := fmt.Sprintf("%s-old", boltPath)
+	if err := os.Rename(boltPath, newBoltDBPath); err != nil {
+		return fmt.Errorf("renaming old database %s to %s: %w", boltPath, newBoltDBPath, err)
+	}
+	fmt.Printf("Old database has been renamed to %s and will no longer be used\n", newBoltDBPath)
+
+	return nil
+}
+
+func getBoltDBPath(runtime *Runtime) string {
+	baseDir := runtime.config.Engine.StaticDir
+	if runtime.storageConfig.TransientStore {
+		baseDir = runtime.config.Engine.TmpDir
+	}
+	return filepath.Join(baseDir, "bolt_state.db")
+}

libpod/sqlite_state.go

@@ -32,6 +32,8 @@ type SQLiteState struct {
 }
 
 const (
+	// Name of the actual database file
+	sqliteDbFilename = "db.sql"
 	// Deal with timezone automatically.
 	sqliteOptionLocation = "_loc=auto"
 	// Force an fsync after each transaction (https://www.sqlite.org/pragma.html#pragma_synchronous).
@@ -44,7 +46,7 @@ const (
 	sqliteOptionCaseSensitiveLike = "&_cslike=TRUE"
 
 	// Assembled sqlite options used when opening the database.
-	sqliteOptions = "db.sql?" +
+	sqliteOptions = sqliteDbFilename + "?" +
 		sqliteOptionLocation +
 		sqliteOptionSynchronous +
 		sqliteOptionForeignKeys +
@@ -57,12 +59,7 @@ func NewSqliteState(runtime *Runtime) (_ State, defErr error) {
 	logrus.Info("Using sqlite as database backend")
 	state := new(SQLiteState)
 
-	basePath := runtime.storageConfig.GraphRoot
-	if runtime.storageConfig.TransientStore {
-		basePath = runtime.storageConfig.RunRoot
-	} else if !runtime.storageSet.StaticDirSet {
-		basePath = runtime.config.Engine.StaticDir
-	}
+	basePath, _ := sqliteStatePath(runtime)
 
 	// c/storage is set up *after* the DB - so even though we use the c/s
 	// root (or, for transient, runroot) dir, we need to make the dir

libpod/sqlite_state_internal.go

@@ -19,6 +19,18 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 )
 
+// Returns two strings. First is base path - directory we'll create in.
+// Second is the filename of the database itself.
+func sqliteStatePath(runtime *Runtime) (string, string) {
+	basePath := runtime.storageConfig.GraphRoot
+	if runtime.storageConfig.TransientStore {
+		basePath = runtime.storageConfig.RunRoot
+	} else if !runtime.storageSet.StaticDirSet {
+		basePath = runtime.config.Engine.StaticDir
+	}
+	return basePath, sqliteDbFilename
+}
+
 func initSQLiteDB(conn *sql.DB) (defErr error) {
 	// Start with a transaction to avoid "database locked" errors.
 	// See https://github.com/mattn/go-sqlite3/issues/274#issuecomment-1429054597

pkg/domain/entities/types/system.go

@@ -63,6 +63,7 @@ type SystemPruneReport struct {
 // cli to migrate runtimes of containers
 type SystemMigrateOptions struct {
 	NewRuntime string
+	MigrateDB  bool
 }
 
 // SystemDfOptions describes the options for getting df information

pkg/domain/infra/abi/system.go

@@ -309,7 +309,7 @@ func (ic *ContainerEngine) Renumber(_ context.Context) error {
 }
 
 func (ic *ContainerEngine) Migrate(_ context.Context, options entities.SystemMigrateOptions) error {
-	return ic.Libpod.Migrate(options.NewRuntime)
+	return ic.Libpod.Migrate(options.NewRuntime, options.MigrateDB)
 }
 
 func unshareEnv(graphroot, runroot string) []string {