Add support for ignoring DDL statements using -- sqlc:ignore

This fix adds tests and some renamings RemoveRollbackStatements -> RemoveIgnoredStatements as now not only rollbacks are ignored.

Add support for ignoring DDL statements using `// sqlc:ignore`.

Author: sgielen

internal/cmd/createdb.go

@@ -85,7 +85,7 @@ func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Option
 		if err != nil {
 			return fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddl = append(ddl, migrations.RemoveIgnoredStatements(string(contents)))
 	}
 
 	now := time.Now().UTC().UnixNano()

internal/cmd/verify.go

@@ -102,7 +102,7 @@ func Verify(ctx context.Context, dir, filename string, opts *Options) error {
 				if err != nil {
 					return fmt.Errorf("read file: %w", err)
 				}
-				ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+				ddl = append(ddl, migrations.RemoveIgnoredStatements(string(contents)))
 			}
 
 			var codegen plugin.GenerateRequest

internal/cmd/vet.go

@@ -419,7 +419,7 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f
 		if err != nil {
 			return "", cleanup, fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddl = append(ddl, migrations.RemoveIgnoredStatements(string(contents)))
 	}
 
 	resp, err := c.Client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{

internal/compiler/compile.go

@@ -37,7 +37,7 @@ func (c *Compiler) parseCatalog(schemas []string) error {
 			merr.Add(filename, "", 0, err)
 			continue
 		}
-		contents := migrations.RemoveRollbackStatements(string(blob))
+		contents := migrations.RemoveIgnoredStatements(string(blob))
 		c.schema = append(c.schema, contents)
 		stmts, err := c.parser.Parse(strings.NewReader(contents))
 		if err != nil {

internal/migrations/migrations.go

@@ -5,29 +5,48 @@ import (
 	"strings"
 )
 
-// Remove all lines after a rollback comment.
+// Remove all lines that should be ignored by sqlc, such as rollback
+// comments or explicit "sqlc:ignore" lines.
 //
 // goose:       -- +goose Down
 // sql-migrate: -- +migrate Down
 // tern:        ---- create above / drop below ----
 // dbmate:      -- migrate:down
-func RemoveRollbackStatements(contents string) string {
+// generic:     `-- sqlc:ignore` until `-- sqlc:ignore end`
+func RemoveIgnoredStatements(contents string) string {
 	s := bufio.NewScanner(strings.NewReader(contents))
 	var lines []string
+	var ignoring bool
 	for s.Scan() {
-		if strings.HasPrefix(s.Text(), "-- +goose Down") {
+		line := s.Text()
+
+		if strings.HasPrefix(line, "-- +goose Down") {
 			break
 		}
-		if strings.HasPrefix(s.Text(), "-- +migrate Down") {
+		if strings.HasPrefix(line, "-- +migrate Down") {
 			break
 		}
-		if strings.HasPrefix(s.Text(), "---- create above / drop below ----") {
+		if strings.HasPrefix(line, "---- create above / drop below ----") {
 			break
 		}
-		if strings.HasPrefix(s.Text(), "-- migrate:down") {
+		if strings.HasPrefix(line, "-- migrate:down") {
 			break
 		}
-		lines = append(lines, s.Text())
+
+		if strings.HasPrefix(line, "-- sqlc:ignore end") {
+			ignoring = false
+			// no need to keep this line in result
+			line = ""
+		} else if strings.HasPrefix(line, "-- sqlc:ignore") {
+			ignoring = true
+		}
+
+		if ignoring {
+			// make this line empty, so that errors are still reported on the
+			// correct line
+			line = ""
+		}
+		lines = append(lines, line)
 	}
 	return strings.Join(lines, "\n")
 }

internal/migrations/migrations_test.go

@@ -10,20 +10,34 @@ const inputGoose = `
 -- +goose Up
 ALTER TABLE archived_jobs ADD COLUMN expires_at TIMESTAMP WITH TIME ZONE;
 
+-- sqlc:ignore
+CREATE TABLE countries (id int);
+CREATE TABLE people (id int);
+-- sqlc:ignore
+
 -- +goose Down
 ALTER TABLE archived_jobs DROP COLUMN expires_at;
 `
 
 const outputGoose = `
 -- +goose Up
 ALTER TABLE archived_jobs ADD COLUMN expires_at TIMESTAMP WITH TIME ZONE;
+
+
+
+
+
 `
 
 const inputMigrate = `
 -- +migrate Up
 -- SQL in section 'Up' is executed when this migration is applied
 CREATE TABLE people (id int);
 
+-- sqlc:ignore
+INVALID SYNTAX HERE IS OK, WE SHOULD IGNORE THIS
+-- sqlc:ignore end
+
 -- +migrate Down
 -- SQL section 'Down' is executed when this migration is rolled back
 DROP TABLE people;
@@ -33,47 +47,66 @@ const outputMigrate = `
 -- +migrate Up
 -- SQL in section 'Up' is executed when this migration is applied
 CREATE TABLE people (id int);
+
+
+
+
 `
 
 const inputTern = `
+-- sqlc:ignore
+As first row also ok, all contents after should be processed
+-- sqlc:ignore end
 -- Write your migrate up statements here
 ALTER TABLE todo RENAME COLUMN done TO is_done;
 ---- create above / drop below ----
 ALTER TABLE todo RENAME COLUMN is_done TO done;
 `
 
 const outputTern = `
+
+
+
 -- Write your migrate up statements here
 ALTER TABLE todo RENAME COLUMN done TO is_done;`
 
 const inputDbmate = `
 -- migrate:up
 CREATE TABLE foo (bar int);
+-- sqlc:ignore
+In up section
+-- sqlc:ignore end
 -- migrate:down
-DROP TABLE foo;`
+DROP TABLE foo;
+-- sqlc:ignore
+In down section
+-- sqlc:ignore end`
 
 const outputDbmate = `
 -- migrate:up
-CREATE TABLE foo (bar int);`
+CREATE TABLE foo (bar int);
+
+
+`
 
-func TestRemoveRollback(t *testing.T) {
-	if diff := cmp.Diff(outputGoose, RemoveRollbackStatements(inputGoose)); diff != "" {
+func TestRemoveIgnored(t *testing.T) {
+	if diff := cmp.Diff(outputGoose, RemoveIgnoredStatements(inputGoose)); diff != "" {
 		t.Errorf("goose migration mismatch:\n%s", diff)
 	}
-	if diff := cmp.Diff(outputMigrate, RemoveRollbackStatements(inputMigrate)); diff != "" {
+	if diff := cmp.Diff(outputMigrate, RemoveIgnoredStatements(inputMigrate)); diff != "" {
 		t.Errorf("sql-migrate migration mismatch:\n%s", diff)
 	}
-	if diff := cmp.Diff(outputTern, RemoveRollbackStatements(inputTern)); diff != "" {
+	if diff := cmp.Diff(outputTern, RemoveIgnoredStatements(inputTern)); diff != "" {
 		t.Errorf("tern migration mismatch:\n%s", diff)
 	}
-	if diff := cmp.Diff(outputDbmate, RemoveRollbackStatements(inputDbmate)); diff != "" {
+	if diff := cmp.Diff(outputDbmate, RemoveIgnoredStatements(inputDbmate)); diff != "" {
 		t.Errorf("dbmate migration mismatch:\n%s", diff)
 	}
 }
 
 func TestRemoveGolangMigrateRollback(t *testing.T) {
 	filenames := map[string]bool{
-		// make sure we let through golang-migrate files that aren't rollbacks
+		// make sure we let through golang-migrate files that aren't ignored
 		"migrations/1.up.sql": false,
 		// make sure we let through other sql files
 		"migrations/2.sql":      false,

internal/sqltest/local/mysql.go

@@ -53,7 +53,7 @@ func MySQL(t *testing.T, migrations []string) string {
 		if err != nil {
 			t.Fatal(err)
 		}
-		seed = append(seed, migrate.RemoveRollbackStatements(string(blob)))
+		seed = append(seed, migrate.RemoveIgnoredStatements(string(blob)))
 	}
 
 	cfg, err := mysql.ParseDSN(dburi)

internal/sqltest/local/postgres.go

@@ -59,7 +59,7 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string {
 			t.Fatal(err)
 		}
 		h.Write(blob)
-		seed = append(seed, migrate.RemoveRollbackStatements(string(blob)))
+		seed = append(seed, migrate.RemoveIgnoredStatements(string(blob)))
 	}
 
 	var name string