From c5589973014f96fc6d7c68cfb646155f2ceac65a Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 12:56:10 -0700 Subject: [PATCH 01/10] test(endtoend): Use Docker to start database servers Remove setup-mysql and setup-postgresql actions --- .github/workflows/ci.yml | 18 ++++----- internal/endtoend/endtoend_test.go | 33 ++++++++++----- internal/sqltest/docker/enabled.go | 13 ++++++ internal/sqltest/docker/mysql.go | 60 ++++++++++++++++++++++++++++ internal/sqltest/docker/postgres.go | 62 +++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 19 deletions(-) create mode 100644 internal/sqltest/docker/enabled.go create mode 100644 internal/sqltest/docker/mysql.go create mode 100644 internal/sqltest/docker/postgres.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d01f019db5..1c3b4b5d66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,15 +53,15 @@ jobs: CGO_ENABLED: ${{ matrix.cgo }} # Start a PostgreSQL server - - uses: sqlc-dev/action-setup-postgres@master - with: - postgres-version: "16" - id: postgres + # - uses: sqlc-dev/action-setup-postgres@master + # with: + # postgres-version: "16" + # id: postgres # Start a MySQL server - - uses: shogo82148/actions-setup-mysql@v1 - with: - mysql-version: "9.0" + # - uses: shogo82148/actions-setup-mysql@v1 + # with: + # mysql-version: "9.0" - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./... @@ -69,8 +69,8 @@ jobs: CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} - MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true - POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable + # MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true + # POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable CGO_ENABLED: ${{ matrix.cgo }} vuln_check: diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index da6d7a405a..311eba9825 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -17,7 +17,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/cmd" "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/opts" - "github.com/sqlc-dev/sqlc/internal/sqltest/local" + "github.com/sqlc-dev/sqlc/internal/sqltest/docker" ) func lineEndings() cmp.Option { @@ -112,6 +112,24 @@ func TestReplay(t *testing.T) { // t.Parallel() ctx := context.Background() + var mysqlURI, postgresURI string + if err := docker.Installed(); err == nil { + { + host, err := docker.StartPostgreSQLServer(ctx) + if err != nil { + t.Fatalf("starting postgresql failed: %s", err) + } + postgresURI = host + } + { + host, err := docker.StartMySQLServer(ctx) + if err != nil { + t.Fatalf("starting mysql failed: %s", err) + } + mysqlURI = host + } + } + contexts := map[string]textContext{ "base": { Mutate: func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} }, @@ -124,13 +142,13 @@ func TestReplay(t *testing.T) { { Name: "postgres", Engine: config.EnginePostgreSQL, - URI: local.PostgreSQLServer(), + URI: postgresURI, }, { Name: "mysql", Engine: config.EngineMySQL, - URI: local.MySQLServer(), + URI: mysqlURI, }, } for i := range c.SQL { @@ -150,13 +168,8 @@ func TestReplay(t *testing.T) { } }, Enabled: func() bool { - if len(os.Getenv("POSTGRESQL_SERVER_URI")) == 0 { - return false - } - if len(os.Getenv("MYSQL_SERVER_URI")) == 0 { - return false - } - return true + err := docker.Installed() + return err == nil }, }, } diff --git a/internal/sqltest/docker/enabled.go b/internal/sqltest/docker/enabled.go new file mode 100644 index 0000000000..28c5e89d17 --- /dev/null +++ b/internal/sqltest/docker/enabled.go @@ -0,0 +1,13 @@ +package docker + +import ( + "fmt" + "os/exec" +) + +func Installed() error { + if _, err := exec.LookPath("docker"); err != nil { + return fmt.Errorf("docker not found: %w", err) + } + return nil +} diff --git a/internal/sqltest/docker/mysql.go b/internal/sqltest/docker/mysql.go new file mode 100644 index 0000000000..4bbae412f8 --- /dev/null +++ b/internal/sqltest/docker/mysql.go @@ -0,0 +1,60 @@ +package docker + +import ( + "context" + "database/sql" + "fmt" + "os/exec" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +func StartMySQLServer(c context.Context) (string, error) { + if err := Installed(); err != nil { + return "", err + } + + ctx, cancel := context.WithTimeout(c, 10*time.Second) + defer cancel() + + cmd := exec.Command("docker", "run", + "--name", "sqlc_sqltest_docker_mysql", + "-e", "MYSQL_ROOT_PASSWORD=mysecretpassword", + "-e", "MYSQL_DATABASE=dinotest", + "-p", "3306:3306", + "-d", + "mysql:8", + ) + + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + return "", err + } + + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + uri := "root:mysecretpassword@/dinotest" + + db, err := sql.Open("mysql", uri) + if err != nil { + return "", fmt.Errorf("sql.Open: %w", err) + } + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout reached: %w", ctx.Err()) + + case <-ticker.C: + // Run your function here + if err := db.PingContext(ctx); err != nil { + continue + } + return uri, nil + } + } +} diff --git a/internal/sqltest/docker/postgres.go b/internal/sqltest/docker/postgres.go new file mode 100644 index 0000000000..065f30515c --- /dev/null +++ b/internal/sqltest/docker/postgres.go @@ -0,0 +1,62 @@ +package docker + +import ( + "context" + "fmt" + "log/slog" + "os/exec" + "time" + + "github.com/jackc/pgx/v5" +) + +func StartPostgreSQLServer(c context.Context) (string, error) { + if err := Installed(); err != nil { + return "", err + } + + ctx, cancel := context.WithTimeout(c, 5*time.Second) + defer cancel() + + cmd := exec.Command("docker", "run", + "--name", "sqlc_sqltest_docker_postgres", + "-e", "POSTGRES_PASSWORD=mysecretpassword", + "-e", "POSTGRES_USER=postgres", + "-p", "5432:5432", + "-d", + "postgres:16", + "-c", "max_connections=200", + ) + + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + return "", err + } + + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable" + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout reached: %w", ctx.Err()) + + case <-ticker.C: + // Run your function here + conn, err := pgx.Connect(ctx, uri) + if err != nil { + slog.Debug("sqltest", "connect", err) + continue + } + if err := conn.Ping(ctx); err != nil { + slog.Error("sqltest", "ping", err) + continue + } + return uri, nil + } + } +} From 0abcc9658a68f5e7eefd00744634a61876cf0394 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 13:01:24 -0700 Subject: [PATCH 02/10] Pull separately --- internal/sqltest/docker/mysql.go | 7 +++++++ internal/sqltest/docker/postgres.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/internal/sqltest/docker/mysql.go b/internal/sqltest/docker/mysql.go index 4bbae412f8..7e6400065e 100644 --- a/internal/sqltest/docker/mysql.go +++ b/internal/sqltest/docker/mysql.go @@ -15,6 +15,13 @@ func StartMySQLServer(c context.Context) (string, error) { return "", err } + { + _, err := exec.Command("docker", "pull", "mysql:8").CombinedOutput() + if err != nil { + return "", fmt.Errorf("docker pull: mysql:8 %w", err) + } + } + ctx, cancel := context.WithTimeout(c, 10*time.Second) defer cancel() diff --git a/internal/sqltest/docker/postgres.go b/internal/sqltest/docker/postgres.go index 065f30515c..6d403e180b 100644 --- a/internal/sqltest/docker/postgres.go +++ b/internal/sqltest/docker/postgres.go @@ -15,6 +15,13 @@ func StartPostgreSQLServer(c context.Context) (string, error) { return "", err } + { + _, err := exec.Command("docker", "pull", "postgres:16").CombinedOutput() + if err != nil { + return "", fmt.Errorf("docker pull: postgres:16 %w", err) + } + } + ctx, cancel := context.WithTimeout(c, 5*time.Second) defer cancel() From 768369fd9f7db1fdcc4ca722157152b9c47e2262 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 13:22:20 -0700 Subject: [PATCH 03/10] sync.Do --- internal/sqltest/docker/mysql.go | 89 +++++++++++++++++----------- internal/sqltest/docker/postgres.go | 90 ++++++++++++++++++----------- 2 files changed, 110 insertions(+), 69 deletions(-) diff --git a/internal/sqltest/docker/mysql.go b/internal/sqltest/docker/mysql.go index 7e6400065e..9e7540b3a5 100644 --- a/internal/sqltest/docker/mysql.go +++ b/internal/sqltest/docker/mysql.go @@ -5,11 +5,15 @@ import ( "database/sql" "fmt" "os/exec" + "sync" "time" _ "github.com/go-sql-driver/mysql" ) +var mysqlSync sync.Once +var mysqlHost string + func StartMySQLServer(c context.Context) (string, error) { if err := Installed(); err != nil { return "", err @@ -22,46 +26,63 @@ func StartMySQLServer(c context.Context) (string, error) { } } - ctx, cancel := context.WithTimeout(c, 10*time.Second) - defer cancel() - - cmd := exec.Command("docker", "run", - "--name", "sqlc_sqltest_docker_mysql", - "-e", "MYSQL_ROOT_PASSWORD=mysecretpassword", - "-e", "MYSQL_DATABASE=dinotest", - "-p", "3306:3306", - "-d", - "mysql:8", - ) - - output, err := cmd.CombinedOutput() - fmt.Println(string(output)) - if err != nil { - return "", err - } + var syncErr error + mysqlSync.Do(func() { + ctx, cancel := context.WithTimeout(c, 10*time.Second) + defer cancel() - // Create a ticker that fires every 10ms - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() + cmd := exec.Command("docker", "run", + "--name", "sqlc_sqltest_docker_mysql", + "-e", "MYSQL_ROOT_PASSWORD=mysecretpassword", + "-e", "MYSQL_DATABASE=dinotest", + "-p", "3306:3306", + "-d", + "mysql:8", + ) - uri := "root:mysecretpassword@/dinotest" + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + syncErr = err + return + } - db, err := sql.Open("mysql", uri) - if err != nil { - return "", fmt.Errorf("sql.Open: %w", err) - } + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + uri := "root:mysecretpassword@/dinotest" - for { - select { - case <-ctx.Done(): - return "", fmt.Errorf("timeout reached: %w", ctx.Err()) + db, err := sql.Open("mysql", uri) + if err != nil { + syncErr = fmt.Errorf("sql.Open: %w", err) + return + } - case <-ticker.C: - // Run your function here - if err := db.PingContext(ctx); err != nil { - continue + for { + select { + case <-ctx.Done(): + syncErr = fmt.Errorf("timeout reached: %w", ctx.Err()) + return + + case <-ticker.C: + // Run your function here + if err := db.PingContext(ctx); err != nil { + continue + } + mysqlHost = uri + return } - return uri, nil } + }) + + if syncErr != nil { + return "", syncErr } + + if mysqlHost == "" { + return "", fmt.Errorf("mysql server setup failed") + } + + return mysqlHost, nil } diff --git a/internal/sqltest/docker/postgres.go b/internal/sqltest/docker/postgres.go index 6d403e180b..989a7fe803 100644 --- a/internal/sqltest/docker/postgres.go +++ b/internal/sqltest/docker/postgres.go @@ -5,11 +5,15 @@ import ( "fmt" "log/slog" "os/exec" + "sync" "time" "github.com/jackc/pgx/v5" ) +var postgresSync sync.Once +var postgresHost string + func StartPostgreSQLServer(c context.Context) (string, error) { if err := Installed(); err != nil { return "", err @@ -22,48 +26,64 @@ func StartPostgreSQLServer(c context.Context) (string, error) { } } - ctx, cancel := context.WithTimeout(c, 5*time.Second) - defer cancel() + var syncErr error + postgresSync.Do(func() { + ctx, cancel := context.WithTimeout(c, 5*time.Second) + defer cancel() - cmd := exec.Command("docker", "run", - "--name", "sqlc_sqltest_docker_postgres", - "-e", "POSTGRES_PASSWORD=mysecretpassword", - "-e", "POSTGRES_USER=postgres", - "-p", "5432:5432", - "-d", - "postgres:16", - "-c", "max_connections=200", - ) + cmd := exec.Command("docker", "run", + "--name", "sqlc_sqltest_docker_postgres", + "-e", "POSTGRES_PASSWORD=mysecretpassword", + "-e", "POSTGRES_USER=postgres", + "-p", "5432:5432", + "-d", + "postgres:16", + "-c", "max_connections=200", + ) - output, err := cmd.CombinedOutput() - fmt.Println(string(output)) - if err != nil { - return "", err - } + output, err := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + syncErr = err + return + } - // Create a ticker that fires every 10ms - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() - uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable" + uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable" - for { - select { - case <-ctx.Done(): - return "", fmt.Errorf("timeout reached: %w", ctx.Err()) + for { + select { + case <-ctx.Done(): + syncErr = fmt.Errorf("timeout reached: %w", ctx.Err()) + return - case <-ticker.C: - // Run your function here - conn, err := pgx.Connect(ctx, uri) - if err != nil { - slog.Debug("sqltest", "connect", err) - continue + case <-ticker.C: + // Run your function here + conn, err := pgx.Connect(ctx, uri) + if err != nil { + slog.Debug("sqltest", "connect", err) + continue + } + if err := conn.Ping(ctx); err != nil { + slog.Error("sqltest", "ping", err) + continue + } + postgresHost = uri + return } - if err := conn.Ping(ctx); err != nil { - slog.Error("sqltest", "ping", err) - continue - } - return uri, nil } + }) + + if syncErr != nil { + return "", syncErr } + + if postgresHost == "" { + return "", fmt.Errorf("postgres server setup failed") + } + + return postgresHost, nil } From a4893b19c6cff0725dd20f42de05235cf701860b Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 13:29:15 -0700 Subject: [PATCH 04/10] Use the new Docker package --- internal/sqltest/local/mysql.go | 15 ++++++++++----- internal/sqltest/local/postgres.go | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/internal/sqltest/local/mysql.go b/internal/sqltest/local/mysql.go index 9c068a39ba..dedd3dfd78 100644 --- a/internal/sqltest/local/mysql.go +++ b/internal/sqltest/local/mysql.go @@ -13,22 +13,27 @@ import ( migrate "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" + "github.com/sqlc-dev/sqlc/internal/sqltest/docker" ) var mysqlSync sync.Once var mysqlPool *sql.DB -func MySQLServer() string { - return os.Getenv("MYSQL_SERVER_URI") -} - func MySQL(t *testing.T, migrations []string) string { ctx := context.Background() t.Helper() dburi := os.Getenv("MYSQL_SERVER_URI") if dburi == "" { - t.Skip("MYSQL_SERVER_URI is empty") + if ierr := docker.Installed(); ierr == nil { + u, err := docker.StartMySQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u + } else { + t.Skip("MYSQL_SERVER_URI is empty") + } } mysqlSync.Do(func() { diff --git a/internal/sqltest/local/postgres.go b/internal/sqltest/local/postgres.go index 7b2c16c40a..feda4cf7ac 100644 --- a/internal/sqltest/local/postgres.go +++ b/internal/sqltest/local/postgres.go @@ -15,6 +15,7 @@ import ( migrate "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/pgx/poolcache" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" + "github.com/sqlc-dev/sqlc/internal/sqltest/docker" ) var flight singleflight.Group @@ -28,17 +29,21 @@ func ReadOnlyPostgreSQL(t *testing.T, migrations []string) string { return postgreSQL(t, migrations, false) } -func PostgreSQLServer() string { - return os.Getenv("POSTGRESQL_SERVER_URI") -} - func postgreSQL(t *testing.T, migrations []string, rw bool) string { ctx := context.Background() t.Helper() dburi := os.Getenv("POSTGRESQL_SERVER_URI") if dburi == "" { - t.Skip("POSTGRESQL_SERVER_URI is empty") + if ierr := docker.Installed(); ierr == nil { + u, err := docker.StartPostgreSQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u + } else { + t.Skip("POSTGRESQL_SERVER_URI is empty") + } } postgresPool, err := cache.Open(ctx, dburi) From 9bdaf614464726f45253111ed2ef8aefb684f32b Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:14:39 -0700 Subject: [PATCH 05/10] Be paranoid about concurrency --- internal/sqltest/docker/enabled.go | 4 ++ internal/sqltest/docker/mysql.go | 104 ++++++++++++++++------------ internal/sqltest/docker/postgres.go | 98 +++++++++++++++----------- 3 files changed, 121 insertions(+), 85 deletions(-) diff --git a/internal/sqltest/docker/enabled.go b/internal/sqltest/docker/enabled.go index 28c5e89d17..e17c0201b2 100644 --- a/internal/sqltest/docker/enabled.go +++ b/internal/sqltest/docker/enabled.go @@ -3,8 +3,12 @@ package docker import ( "fmt" "os/exec" + + "golang.org/x/sync/singleflight" ) +var flight singleflight.Group + func Installed() error { if _, err := exec.LookPath("docker"); err != nil { return fmt.Errorf("docker not found: %w", err) diff --git a/internal/sqltest/docker/mysql.go b/internal/sqltest/docker/mysql.go index 9e7540b3a5..39a1af6160 100644 --- a/internal/sqltest/docker/mysql.go +++ b/internal/sqltest/docker/mysql.go @@ -5,84 +5,100 @@ import ( "database/sql" "fmt" "os/exec" - "sync" + "strings" "time" _ "github.com/go-sql-driver/mysql" ) -var mysqlSync sync.Once var mysqlHost string func StartMySQLServer(c context.Context) (string, error) { if err := Installed(); err != nil { return "", err } + if mysqlHost != "" { + return mysqlHost, nil + } + value, err, _ := flight.Do("mysql", func() (interface{}, error) { + host, err := startMySQLServer(c) + if err != nil { + return "", err + } + mysqlHost = host + return host, nil + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} +func startMySQLServer(c context.Context) (string, error) { { - _, err := exec.Command("docker", "pull", "mysql:8").CombinedOutput() + _, err := exec.Command("docker", "pull", "mysql:9").CombinedOutput() if err != nil { - return "", fmt.Errorf("docker pull: mysql:8 %w", err) + return "", fmt.Errorf("docker pull: mysql:9 %w", err) } } - var syncErr error - mysqlSync.Do(func() { - ctx, cancel := context.WithTimeout(c, 10*time.Second) - defer cancel() + var exists bool + { + cmd := exec.Command("docker", "container", "inspect", "sqlc_sqltest_docker_mysql") + // This means we've already started the container + exists = cmd.Run() == nil + } + if !exists { cmd := exec.Command("docker", "run", "--name", "sqlc_sqltest_docker_mysql", "-e", "MYSQL_ROOT_PASSWORD=mysecretpassword", "-e", "MYSQL_DATABASE=dinotest", "-p", "3306:3306", "-d", - "mysql:8", + "mysql:9", ) output, err := cmd.CombinedOutput() fmt.Println(string(output)) - if err != nil { - syncErr = err - return - } - // Create a ticker that fires every 10ms - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() + msg := `Conflict. The container name "/sqlc_sqltest_docker_mysql" is already in use by container` + if !strings.Contains(string(output), msg) && err != nil { + return "", err + } + } - uri := "root:mysecretpassword@/dinotest" + ctx, cancel := context.WithTimeout(c, 10*time.Second) + defer cancel() - db, err := sql.Open("mysql", uri) - if err != nil { - syncErr = fmt.Errorf("sql.Open: %w", err) - return - } + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() - for { - select { - case <-ctx.Done(): - syncErr = fmt.Errorf("timeout reached: %w", ctx.Err()) - return - - case <-ticker.C: - // Run your function here - if err := db.PingContext(ctx); err != nil { - continue - } - mysqlHost = uri - return - } - } - }) + uri := "root:mysecretpassword@/dinotest?multiStatements=true&parseTime=true" - if syncErr != nil { - return "", syncErr + db, err := sql.Open("mysql", uri) + if err != nil { + return "", fmt.Errorf("sql.Open: %w", err) } - if mysqlHost == "" { - return "", fmt.Errorf("mysql server setup failed") - } + defer db.Close() + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout reached: %w", ctx.Err()) - return mysqlHost, nil + case <-ticker.C: + // Run your function here + if err := db.PingContext(ctx); err != nil { + continue + } + return uri, nil + } + } } diff --git a/internal/sqltest/docker/postgres.go b/internal/sqltest/docker/postgres.go index 989a7fe803..1b2d842c70 100644 --- a/internal/sqltest/docker/postgres.go +++ b/internal/sqltest/docker/postgres.go @@ -5,20 +5,40 @@ import ( "fmt" "log/slog" "os/exec" - "sync" + "strings" "time" "github.com/jackc/pgx/v5" ) -var postgresSync sync.Once var postgresHost string func StartPostgreSQLServer(c context.Context) (string, error) { if err := Installed(); err != nil { return "", err } + if postgresHost != "" { + return postgresHost, nil + } + value, err, _ := flight.Do("postgresql", func() (interface{}, error) { + host, err := startPostgreSQLServer(c) + if err != nil { + return "", err + } + postgresHost = host + return host, err + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} +func startPostgreSQLServer(c context.Context) (string, error) { { _, err := exec.Command("docker", "pull", "postgres:16").CombinedOutput() if err != nil { @@ -26,11 +46,16 @@ func StartPostgreSQLServer(c context.Context) (string, error) { } } - var syncErr error - postgresSync.Do(func() { - ctx, cancel := context.WithTimeout(c, 5*time.Second) - defer cancel() + uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable" + + var exists bool + { + cmd := exec.Command("docker", "container", "inspect", "sqlc_sqltest_docker_postgres") + // This means we've already started the container + exists = cmd.Run() == nil + } + if !exists { cmd := exec.Command("docker", "run", "--name", "sqlc_sqltest_docker_postgres", "-e", "POSTGRES_PASSWORD=mysecretpassword", @@ -43,47 +68,38 @@ func StartPostgreSQLServer(c context.Context) (string, error) { output, err := cmd.CombinedOutput() fmt.Println(string(output)) - if err != nil { - syncErr = err - return + + msg := `Conflict. The container name "/sqlc_sqltest_docker_postgres" is already in use by container` + if !strings.Contains(string(output), msg) && err != nil { + return "", err } + } - // Create a ticker that fires every 10ms - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() + ctx, cancel := context.WithTimeout(c, 5*time.Second) + defer cancel() - uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable" + // Create a ticker that fires every 10ms + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() - for { - select { - case <-ctx.Done(): - syncErr = fmt.Errorf("timeout reached: %w", ctx.Err()) - return + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout reached: %w", ctx.Err()) - case <-ticker.C: - // Run your function here - conn, err := pgx.Connect(ctx, uri) - if err != nil { - slog.Debug("sqltest", "connect", err) - continue - } - if err := conn.Ping(ctx); err != nil { - slog.Error("sqltest", "ping", err) - continue - } - postgresHost = uri - return + case <-ticker.C: + // Run your function here + conn, err := pgx.Connect(ctx, uri) + if err != nil { + slog.Debug("sqltest", "connect", err) + continue + } + defer conn.Close(ctx) + if err := conn.Ping(ctx); err != nil { + slog.Error("sqltest", "ping", err) + continue } + return uri, nil } - }) - - if syncErr != nil { - return "", syncErr } - - if postgresHost == "" { - return "", fmt.Errorf("postgres server setup failed") - } - - return postgresHost, nil } From f69c738dadcb47c5b433d854276334f1fba4234b Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:17:29 -0700 Subject: [PATCH 06/10] Test if things work on Windows and macOS --- .github/workflows/ci.yml | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c3b4b5d66..6fffb68793 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,20 +9,19 @@ jobs: strategy: matrix: # Disabling windows builds while we fix installing PostgreSQL 16 - # os: [ubuntu-22.04, macos-14, windows-2022] - os: [ubuntu-22.04, macos-15] + os: [ubuntu-22.04, macos-14, windows-2022] cgo: ['1', '0'] # Workaround no native support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 - isMain: - - ${{ github.ref == 'refs/heads/main' }} - exclude: - - isMain: false - include: - - os: ubuntu-22.04 - cgo: '1' - - os: ubuntu-22.04 - cgo: '0' + # isMain: + # - ${{ github.ref == 'refs/heads/main' }} + # exclude: + # - isMain: false + # include: + # - os: ubuntu-22.04 + # cgo: '1' + # - os: ubuntu-22.04 + # cgo: '0' name: test ${{ matrix.os }} cgo=${{ matrix.cgo }} runs-on: ${{ matrix.os }} @@ -52,25 +51,12 @@ jobs: env: CGO_ENABLED: ${{ matrix.cgo }} - # Start a PostgreSQL server - # - uses: sqlc-dev/action-setup-postgres@master - # with: - # postgres-version: "16" - # id: postgres - - # Start a MySQL server - # - uses: shogo82148/actions-setup-mysql@v1 - # with: - # mysql-version: "9.0" - - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./... env: CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} - # MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true - # POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable CGO_ENABLED: ${{ matrix.cgo }} vuln_check: From 2c69f10e3386e1f5652d492f69d79c9b2d85e8a5 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:26:35 -0700 Subject: [PATCH 07/10] don't run tests on windows --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fffb68793..d9a67692a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: # Disabling windows builds while we fix installing PostgreSQL 16 - os: [ubuntu-22.04, macos-14, windows-2022] + os: [ubuntu-24.04, macos-14, windows-2022] cgo: ['1', '0'] # Workaround no native support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 @@ -18,9 +18,9 @@ jobs: # exclude: # - isMain: false # include: - # - os: ubuntu-22.04 + # - os: ubuntu-24.04 # cgo: '1' - # - os: ubuntu-22.04 + # - os: ubuntu-24.04 # cgo: '0' name: test ${{ matrix.os }} cgo=${{ matrix.cgo }} runs-on: ${{ matrix.os }} @@ -53,6 +53,7 @@ jobs: - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./... + if: ${{ matrix.os }} != "windows-2022" env: CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} @@ -60,7 +61,7 @@ jobs: CGO_ENABLED: ${{ matrix.cgo }} vuln_check: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: From 153be0daa7bd5916534cfecf82d46923a0bdcfe2 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:32:51 -0700 Subject: [PATCH 08/10] Only run tests on Linux --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9a67692a8..94e655787f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,8 @@ jobs: strategy: matrix: # Disabling windows builds while we fix installing PostgreSQL 16 - os: [ubuntu-24.04, macos-14, windows-2022] - cgo: ['1', '0'] + # os: [ubuntu-24.04, macos-14, windows-2022] + os: [ubuntu-24.04] # Workaround no native support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 # isMain: @@ -19,10 +19,7 @@ jobs: # - isMain: false # include: # - os: ubuntu-24.04 - # cgo: '1' - # - os: ubuntu-24.04 - # cgo: '0' - name: test ${{ matrix.os }} cgo=${{ matrix.cgo }} + name: test ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: @@ -43,13 +40,13 @@ jobs: - name: install ./... run: go install ./... env: - CGO_ENABLED: ${{ matrix.cgo }} + CGO_ENABLED: "0" - name: build internal/endtoend run: go build ./... working-directory: internal/endtoend/testdata env: - CGO_ENABLED: ${{ matrix.cgo }} + CGO_ENABLED: "0" - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./... @@ -58,7 +55,7 @@ jobs: CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} - CGO_ENABLED: ${{ matrix.cgo }} + CGO_ENABLED: "0" vuln_check: runs-on: ubuntu-24.04 From 76bd5857ccc94f4c348a610fb6abaa234e0b6cba Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:34:32 -0700 Subject: [PATCH 09/10] Add a build workflow --- .github/workflows/build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..96d26bdd1a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,19 @@ +name: build +on: + workflow_dispatch: +jobs: + build: + strategy: + matrix: + os: [ubuntu-24.04, macos-14, windows-2022] + name: build ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v5 + with: + go-version: '1.25.0' + - name: install ./... + run: go build ./... + env: + CGO_ENABLED: "0" From 5ded7d917159c9af950c9f1a53bb4ccffd30d178 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 25 Aug 2025 14:39:48 -0700 Subject: [PATCH 10/10] Split out build and test --- .github/workflows/ci.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e655787f..2c3d89fc5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,23 +5,25 @@ on: - main pull_request: jobs: - test: + build: strategy: matrix: - # Disabling windows builds while we fix installing PostgreSQL 16 - # os: [ubuntu-24.04, macos-14, windows-2022] - os: [ubuntu-24.04] - # Workaround no native support for conditional matrix items - # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 - # isMain: - # - ${{ github.ref == 'refs/heads/main' }} - # exclude: - # - isMain: false - # include: - # - os: ubuntu-24.04 - name: test ${{ matrix.os }} - runs-on: ${{ matrix.os }} - + goos: [darwin, linux, windows] + goarch: [amd64, arm64] + name: build ${{ matrix.goos }}/${{ matrix.goarch }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v5 + with: + go-version: '1.25.0' + - run: go build ./... + env: + CGO_ENABLED: "0" + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + test: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v5 - uses: actions/setup-go@v5