feat(migration): sqlite migration improvements.
Some functions split to be more cohesive, custom PostSteps are stored as pointers to allow their retrieval and parameters passing in runtime if needed (some extra work is dropped and TBD when needed)
This commit is contained in:
Ivan Belyakov
IvanBelyakoff
parent
d106b449b6
commit
aa3d33a58f
|
|
@ -20,7 +20,7 @@ import (
|
|||
|
||||
const nodeCfgMigrationDate = 1640111208
|
||||
|
||||
var customSteps = []sqlite.PostStep{
|
||||
var customSteps = []*sqlite.PostStep{
|
||||
{Version: 1674136690, CustomMigration: migrateEnsUsernames},
|
||||
{Version: 1686048341, CustomMigration: migrateWalletJSONBlobs, RollBackVersion: 1686041510},
|
||||
{Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075},
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ func TestMigrateWalletJsonBlobs(t *testing.T) {
|
|||
err = insertTestTransaction(9, uniswapV3TxTestData, uniswapV3ReceiptTestData, uniswapV3LogTestData, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
failMigrationSteps := []sqlite.PostStep{
|
||||
failMigrationSteps := []*sqlite.PostStep{
|
||||
{
|
||||
Version: customSteps[1].Version,
|
||||
CustomMigration: func(sqlTx *sql.Tx) error {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
|
||||
// Migrate applies migrations.
|
||||
// see Migrate in vendor/status-go/sqlite/migrate.go
|
||||
func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
|
||||
func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
|
||||
return sqlite.Migrate(db, bindata.Resource(
|
||||
AssetNames(),
|
||||
func(name string) ([]byte, error) {
|
||||
|
|
@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
|
|||
}
|
||||
|
||||
// MigrateTo is used for testing purposes
|
||||
func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
|
||||
func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error {
|
||||
return sqlite.Migrate(db, bindata.Resource(
|
||||
AssetNames(),
|
||||
func(name string) ([]byte, error) {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
|
||||
// Migrate applies migrations.
|
||||
// see Migrate in vendor/status-go/sqlite/migrate.go
|
||||
func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
|
||||
func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
|
||||
return sqlite.Migrate(db, bindata.Resource(
|
||||
AssetNames(),
|
||||
func(name string) ([]byte, error) {
|
||||
|
|
|
|||
|
|
@ -10,9 +10,11 @@ import (
|
|||
bindata "github.com/status-im/migrate/v4/source/go_bindata"
|
||||
)
|
||||
|
||||
type CustomMigrationFunc func(tx *sql.Tx) error
|
||||
|
||||
type PostStep struct {
|
||||
Version uint
|
||||
CustomMigration func(tx *sql.Tx) error
|
||||
CustomMigration CustomMigrationFunc
|
||||
RollBackVersion uint
|
||||
}
|
||||
|
||||
|
|
@ -36,7 +38,7 @@ var migrationTable = "status_go_" + sqlcipher.DefaultMigrationsTable
|
|||
//
|
||||
// untilVersion, for testing purposes optional parameter, can be used to limit the migration to a specific version.
|
||||
// Pass nil to migrate to the latest available version.
|
||||
func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep, untilVersion *uint) error {
|
||||
func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []*PostStep, untilVersion *uint) error {
|
||||
source, err := bindata.WithInstance(resources)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create bindata migration source: %w", err)
|
||||
|
|
@ -82,7 +84,7 @@ func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep,
|
|||
|
||||
// runCustomMigrations performs source migrations from current to each custom steps, then runs custom migration callback
|
||||
// until it executes all custom migrations or an error occurs and it tries to rollback to RollBackVersion if > 0.
|
||||
func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep, customIndex int, untilVersion *uint) error {
|
||||
func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []*PostStep, customIndex int, untilVersion *uint) error {
|
||||
for customIndex < len(customSteps) && (untilVersion == nil || customSteps[customIndex].Version <= *untilVersion) {
|
||||
customStep := customSteps[customIndex]
|
||||
|
||||
|
|
@ -99,7 +101,8 @@ func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep,
|
|||
return nil
|
||||
}
|
||||
|
||||
func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate) error {
|
||||
func runCustomMigrationStep(db *sql.DB, customStep *PostStep, m *migrate.Migrate) error {
|
||||
|
||||
sqlTx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
|
|
@ -116,7 +119,7 @@ func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate)
|
|||
return nil
|
||||
}
|
||||
|
||||
func rollbackCustomMigration(m *migrate.Migrate, customStep PostStep, customErr error) error {
|
||||
func rollbackCustomMigration(m *migrate.Migrate, customStep *PostStep, customErr error) error {
|
||||
if customStep.RollBackVersion > 0 {
|
||||
err := m.Migrate(customStep.RollBackVersion)
|
||||
newV, _, _ := m.Version()
|
||||
|
|
@ -135,7 +138,8 @@ func runRemainingMigrations(m *migrate.Migrate, untilVersion *uint) error {
|
|||
}
|
||||
} else {
|
||||
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
|
||||
return fmt.Errorf("failed to migrate up: %w", err)
|
||||
ver, _, _ := m.Version()
|
||||
return fmt.Errorf("failed to migrate up: %w, current version: %d", err, ver)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ const (
|
|||
InMemoryPath = ":memory:"
|
||||
V4CipherPageSize = 8192
|
||||
V3CipherPageSize = 1024
|
||||
sqlMainDatabase = "main"
|
||||
)
|
||||
|
||||
// DecryptDB completely removes the encryption from the db
|
||||
|
|
@ -58,7 +59,22 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber
|
|||
defer onEnd()
|
||||
}
|
||||
|
||||
_, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
|
||||
attachedDbName := "encrypted"
|
||||
err := attachDatabaseWithDefaultSettings(db, encryptedPath, attachedDbName, key, kdfIterationsNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(fmt.Sprintf(`SELECT sqlcipher_export('%s')`, attachedDbName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = db.Exec(fmt.Sprintf(`DETACH DATABASE %s`, attachedDbName))
|
||||
return err
|
||||
}
|
||||
|
||||
func attachDatabaseWithDefaultSettings(db *sql.DB, attachedDbPath string, attachedDbName string, key string, kdfIterationsNumber int) error {
|
||||
_, err := db.Exec(fmt.Sprintf(`ATTACH DATABASE '%s' AS %s KEY '%s'`, attachedDbPath, attachedDbName, key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -67,31 +83,39 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber
|
|||
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
|
||||
}
|
||||
|
||||
_, err = db.Exec(fmt.Sprintf("PRAGMA encrypted.kdf_iter = '%d'", kdfIterationsNumber))
|
||||
if _, err := db.Exec(fmt.Sprintf(`PRAGMA %s.busy_timeout = 60000`, attachedDbName)); err != nil {
|
||||
return errors.New("failed to set `busy_timeout` pragma on attached db")
|
||||
}
|
||||
|
||||
return setDatabaseCipherSettings(db, kdfIterationsNumber, attachedDbName)
|
||||
}
|
||||
|
||||
func setDatabaseCipherSettings(db *sql.DB, kdfIterationsNumber int, dbNameOpt ...string) error {
|
||||
dbName := sqlMainDatabase
|
||||
if len(dbNameOpt) > 0 {
|
||||
dbName = dbNameOpt[0]
|
||||
}
|
||||
|
||||
_, err := db.Exec(fmt.Sprintf("PRAGMA %s.kdf_iter = '%d'", dbName, kdfIterationsNumber))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_page_size = %d", dbName, V4CipherPageSize)); err != nil {
|
||||
fmt.Println("failed to set cipher_page_size pragma")
|
||||
return err
|
||||
}
|
||||
if _, err := db.Exec("PRAGMA encrypted.cipher_hmac_algorithm = HMAC_SHA1"); err != nil {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_hmac_algorithm = HMAC_SHA1", dbName)); err != nil {
|
||||
fmt.Println("failed to set cipher_hmac_algorithm pragma")
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := db.Exec("PRAGMA encrypted.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"); err != nil {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1", dbName)); err != nil {
|
||||
fmt.Println("failed to set cipher_kdf_algorithm pragma")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`SELECT sqlcipher_export('encrypted')`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = db.Exec(`DETACH DATABASE encrypted`)
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncryptDB takes a plaintext database and adds encryption
|
||||
|
|
@ -140,7 +164,7 @@ func buildSqlcipherDSN(path string) (string, error) {
|
|||
return path + queryOperator + "_txlock=immediate", nil
|
||||
}
|
||||
|
||||
func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) {
|
||||
func openDB(path string, key string, kdfIterationsNumber int, cipherPageSize int) (*sql.DB, error) {
|
||||
driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers()))
|
||||
sql.Register(driverName, &sqlcipher.SQLiteDriver{
|
||||
ConnectHook: func(conn *sqlcipher.SQLiteConn) error {
|
||||
|
|
@ -156,7 +180,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
|
|||
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
|
||||
}
|
||||
|
||||
if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil {
|
||||
if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", cipherPageSize), nil); err != nil {
|
||||
fmt.Println("failed to set cipher_page_size pragma")
|
||||
return err
|
||||
}
|
||||
|
|
@ -189,7 +213,6 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
|
|||
})
|
||||
|
||||
dsn, err := buildSqlcipherDSN(path)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -223,7 +246,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
|
|||
return db, nil
|
||||
}
|
||||
|
||||
// OpenDB opens not-encrypted database.
|
||||
// OpenDB opens encrypted database.
|
||||
func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
|
||||
return openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,7 @@ func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int
|
|||
return InitializeDB(path, password, kdfIterationsNumber)
|
||||
}
|
||||
|
||||
var walletCustomSteps = []sqlite.PostStep{
|
||||
}
|
||||
var walletCustomSteps = []*sqlite.PostStep{}
|
||||
|
||||
func doMigration(db *sql.DB) error {
|
||||
// Run all the new migrations
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
|
||||
// Migrate applies migrations.
|
||||
// see Migrate in vendor/status-go/sqlite/migrate.go
|
||||
func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
|
||||
func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
|
||||
return sqlite.Migrate(db, bindata.Resource(
|
||||
AssetNames(),
|
||||
func(name string) ([]byte, error) {
|
||||
|
|
@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
|
|||
}
|
||||
|
||||
// MigrateTo is used for testing purposes
|
||||
func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
|
||||
func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error {
|
||||
return sqlite.Migrate(db, bindata.Resource(
|
||||
AssetNames(),
|
||||
func(name string) ([]byte, error) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue