cgosqlite,sqlite: expose incremental API and add documentation

It is likely useful to report progress information and offer abort for
large databases, so expose this in the high level API in a somewhat safe
way.

A lock was not introduced into the driver directly at this time because
doing so is much more complex, particularly as Stmt.DBHandle would need
to recover that lock - essnetially requiring global state with further
synchronization.

Author: raggi

cgosqlite/cgosqlite.go

@@ -182,30 +182,33 @@ type backup struct {
 	backup *C.sqlite3_backup
 }
 
-func (b *backup) Step(numPages int) (more bool, err error) {
+func (b *backup) Step(numPages int) (more bool, remaining, pageCount int, err error) {
 	res := C.sqlite3_backup_step(b.backup, C.int(numPages))
-	if res == C.SQLITE_DONE {
-		return false, nil
-	}
-	if res == C.SQLITE_OK {
-		return true, nil
+
+	// It is not safe to call remaining and pagecount concurrently with step, so
+	// instead just return them each time.
+	remaining = int(C.sqlite3_backup_remaining(b.backup))
+	pageCount = int(C.sqlite3_backup_pagecount(b.backup))
+
+	more = true
+	switch res {
+	case C.SQLITE_OK, C.SQLITE_BUSY, C.SQLITE_LOCKED:
+		more = true
+	default:
+		more = false
 	}
-	return false, errCode(res)
+
+	return more, remaining, pageCount, errCode(res)
 }
 
 func (b *backup) Finish() error {
 	res := C.sqlite3_backup_finish(b.backup)
+	if res == C.SQLITE_OK {
+		return nil
+	}
 	return errCode(res)
 }
 
-func (b *backup) Remaining() int {
-	return int(C.sqlite3_backup_remaining(b.backup))
-}
-
-func (b *backup) PageCount() int {
-	return int(C.sqlite3_backup_pagecount(b.backup))
-}
-
 func (db *DB) Prepare(query string, prepFlags sqliteh.PrepareFlags) (stmt sqliteh.Stmt, remainingQuery string, err error) {
 	csql := C.CString(query)
 	defer C.free(unsafe.Pointer(csql))

sqlite.go

@@ -855,23 +855,73 @@ func DB(sqlconn SQLConn, fn func(sqliteh.DB) error) error {
 	})
 }
 
-// Backup backups the specified database from srcConn to dstConn.
-func Backup(dstConn SQLConn, dstSchema string, srcConn SQLConn, srcSchema string) error {
-	return DB(dstConn, func(dst sqliteh.DB) error {
+// Backup holds an in-progress backup context.
+type Backup struct {
+	backup sqliteh.Backup
+	src    sqliteh.DB
+	dst    sqliteh.DB
+}
+
+// NewBackup starts a new backup operation that will read from the SQLite
+// database srcConn, schema srcSchema, and write to the database dstConn, schema
+// dstSchema.
+// The database owned by dstConn will be locked for the duration, and must not
+// be modified by other connections or processes.
+// The database owned by srcConn will be read-locked during each call to Step,
+// but can otherwise be used normally. Applications must arrange to ensure that
+// there is mutual exclusion between queries and step calls on the source
+// connection.
+// If a different connection alters the source database during the backup, Step
+// will restart the backup process from the beginning, however if the source
+// connection alters the database, the backup can continue and will include
+// pages affected by the concurrent transactions.
+// Finish must be called on the returned backup object in order to free
+// resources consumed by the backup operation, even if errors occur during steps
+// of the backup process. Finish can also be called any time that Step is not
+// running in order to abort the backup.
+func NewBackup(dstConn SQLConn, dstSchema string, srcConn SQLConn, srcSchema string) (*Backup, error) {
+	var b Backup
+	err := DB(dstConn, func(dst sqliteh.DB) error {
 		return DB(srcConn, func(src sqliteh.DB) error {
-			b, err := dst.BackupInit(dstSchema, src, srcSchema)
-			if err != nil {
-				return errWithMsg(dst, err, "Backup")
-			}
-			more := true
-			for more {
-				more, err = b.Step(1024)
-				if err != nil {
-					b.Finish()
-					return errWithMsg(dst, err, "Step")
-				}
-			}
-			return errWithMsg(dst, b.Finish(), "Finish")
+			var err error
+			b.src = src
+			b.dst = dst
+			b.backup, err = dst.BackupInit(dstSchema, src, srcSchema)
+			return errWithMsg(dst, err, "Backup")
 		})
 	})
+	return &b, err
+}
+
+// Step makes incremental progress toward a complete online backup. It performs
+// at most numPages of copies from the source database to the target database.
+//
+// Step may be called in between other queries on the source connection, so as
+// to concurrently to service traffic, however Step must not be called in
+// parallel with other queries on the source connection.
+//
+// Step may return more=true and non-fatal errors of either SQLITE_BUSY or
+// SQLITE_LOCKED, however either of these errors being returned likely indicate
+// that an external writer has modified the source database and there will be a
+// side effect that the backup will restart from the beginning on the next call
+// to Step.
+//
+// Progress is reported by the `remaining` and `pageCount` values. remaining is
+// the number of pages left to copy, and pageCount is the current number of
+// pages in total that must be copied. pageCount may change in size due to
+// writes that occur during the backup process.
+func (b *Backup) Step(numPages int) (more bool, remaining, pageCount int, err error) {
+	more, remaining, pageCount, err = b.backup.Step(numPages)
+	err = errWithMsg(b.dst, err, "Step")
+	return
+}
+
+// Finish frees up the backup object and any resources it consumes. It must be
+// called even if errors occured calling Step. If Step reports a fatal error,
+// Finish will also return the same error. Finish can be called at any time to
+// abort a backup operation early.
+func (b *Backup) Finish() error {
+	err := b.backup.Finish()
+	err = errWithMsg(b.dst, err, "Finish")
+	return err
 }

sqlite_test.go

@@ -813,7 +813,30 @@ func TestBackup(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer dstConn.Close()
-	if err := Backup(dstConn, "main", srcConn, "main"); err != nil {
+
+	var backup = func(dstConn *sql.Conn, dstName string, srcConn *sql.Conn, srcName string) error {
+		t.Helper()
+		b, err := NewBackup(dstConn, dstName, srcConn, srcName)
+		if err != nil {
+			return err
+		}
+		var (
+			more      = true
+			remaining int
+			pageCount int
+		)
+		for more {
+			more, remaining, pageCount, err = b.Step(1024)
+			t.Logf("backup step: more=%v, remaining=%d, pageCount=%d", more, remaining, pageCount)
+			if err != nil {
+				t.Errorf("backup step: %v", err)
+				more = false
+			}
+		}
+		return b.Finish()
+	}
+
+	if err := backup(dstConn, "main", srcConn, "main"); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := dstConn.ExecContext(ctx, "ATTACH 'file:dst2?mode=memory' AS dst2;"); err != nil {
@@ -823,7 +846,7 @@ func TestBackup(t *testing.T) {
 	if err := dstConn.QueryRowContext(ctx, "SELECT count(*) FROM t1").Scan(&count); err != nil || count != 1 {
 		t.Fatalf("err=%v, count=%d", err, count)
 	}
-	if err := Backup(dstConn, "dst2", srcConn, "src2"); err != nil {
+	if err := backup(dstConn, "dst2", srcConn, "src2"); err != nil {
 		t.Fatal(err)
 	}
 	count = 0

sqliteh/sqliteh.go

@@ -172,11 +172,9 @@ type Stmt interface {
 // https://www.sqlite.org/c3ref/backup_finish.html
 type Backup interface {
 	// Step is called repeatedly to transfer data between the two DBs.
-	Step(numPages int) (more bool, err error)
+	Step(numPages int) (more bool, remaining, pageCount int, err error)
 	// Finish releases all resources associated with the Backup.
 	Finish() error
-	Remaining() int
-	PageCount() int
 }
 
 // ColumnType are constants for each of the SQLite datatypes.