diff --git a/internal/endtoend/CLAUDE.md b/internal/endtoend/CLAUDE.md new file mode 100644 index 0000000000..b9c995c9df --- /dev/null +++ b/internal/endtoend/CLAUDE.md @@ -0,0 +1,117 @@ +# End-to-End Tests - Native Database Setup + +This document describes how to set up MySQL and PostgreSQL for running end-to-end tests in environments without Docker, particularly when using an HTTP proxy. + +## Overview + +The end-to-end tests support three methods for connecting to databases: + +1. **Environment Variables**: Set `POSTGRESQL_SERVER_URI` and `MYSQL_SERVER_URI` directly +2. **Docker**: Automatically starts containers via the docker package +3. **Native Installation**: Starts existing database services on Linux + +## Installing Databases with HTTP Proxy + +In environments where DNS doesn't work directly but an HTTP proxy is available (e.g., some CI environments), you need to configure apt to use the proxy before installing packages. + +### Configure apt Proxy + +```bash +# Check if HTTP_PROXY is set +echo $HTTP_PROXY + +# Configure apt to use the proxy +sudo tee /etc/apt/apt.conf.d/99proxy << EOF +Acquire::http::Proxy "$HTTP_PROXY"; +Acquire::https::Proxy "$HTTPS_PROXY"; +EOF + +# Update package lists +sudo apt-get update -qq +``` + +### Install PostgreSQL + +```bash +# Install PostgreSQL +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql postgresql-contrib + +# Start the service +sudo service postgresql start + +# Set password for postgres user +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + +# Configure pg_hba.conf for password authentication +# Find the hba_file location: +sudo -u postgres psql -t -c "SHOW hba_file;" + +# Add md5 authentication for localhost (add to the beginning of pg_hba.conf): +# host all all 127.0.0.1/32 md5 + +# Reload PostgreSQL +sudo service postgresql reload +``` + +### Install MySQL + +```bash +# Pre-configure MySQL root password +echo "mysql-server mysql-server/root_password password mysecretpassword" | sudo debconf-set-selections +echo "mysql-server mysql-server/root_password_again password mysecretpassword" | sudo debconf-set-selections + +# Install MySQL +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server + +# Start the service +sudo service mysql start + +# Verify connection +mysql -uroot -pmysecretpassword -e "SELECT 1;" +``` + +## Expected Database Credentials + +The native database support expects the following credentials: + +### PostgreSQL +- **URI**: `postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable` +- **User**: `postgres` +- **Password**: `postgres` +- **Port**: `5432` + +### MySQL +- **URI**: `root:mysecretpassword@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true` +- **User**: `root` +- **Password**: `mysecretpassword` +- **Port**: `3306` + +## Running Tests + +```bash +# Run end-to-end tests +go test -v -run TestReplay -timeout 20m ./internal/endtoend/... + +# With verbose logging +go test -v -run TestReplay -timeout 20m ./internal/endtoend/... 2>&1 | tee test.log +``` + +## Troubleshooting + +### apt-get times out or fails +- Ensure HTTP proxy is configured in `/etc/apt/apt.conf.d/99proxy` +- Check that the proxy URL is correct: `echo $HTTP_PROXY` +- Try running `sudo apt-get update` first to verify connectivity + +### MySQL connection refused +- Check if MySQL is running: `sudo service mysql status` +- Verify the password: `mysql -uroot -pmysecretpassword -e "SELECT 1;"` +- Check if MySQL is listening on TCP: `netstat -tlnp | grep 3306` + +### PostgreSQL authentication failed +- Verify pg_hba.conf has md5 authentication for localhost +- Check password: `PGPASSWORD=postgres psql -h localhost -U postgres -c "SELECT 1;"` +- Reload PostgreSQL after config changes: `sudo service postgresql reload` + +### DNS resolution fails +This is expected in some environments. Configure apt proxy as shown above. diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index 537307e453..cd7072a7a9 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -18,6 +18,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) func lineEndings() cmp.Option { @@ -113,23 +114,63 @@ func TestReplay(t *testing.T) { ctx := context.Background() var mysqlURI, postgresURI string - if err := docker.Installed(); err == nil { - { - host, err := docker.StartPostgreSQLServer(ctx) - if err != nil { - t.Fatalf("starting postgresql failed: %s", err) + + // First, check environment variables + if uri := os.Getenv("POSTGRESQL_SERVER_URI"); uri != "" { + postgresURI = uri + } + if uri := os.Getenv("MYSQL_SERVER_URI"); uri != "" { + mysqlURI = uri + } + + // Try Docker for any missing databases + if postgresURI == "" || mysqlURI == "" { + if err := docker.Installed(); err == nil { + if postgresURI == "" { + host, err := docker.StartPostgreSQLServer(ctx) + if err != nil { + t.Logf("docker postgresql startup failed: %s", err) + } else { + postgresURI = host + } + } + if mysqlURI == "" { + host, err := docker.StartMySQLServer(ctx) + if err != nil { + t.Logf("docker mysql startup failed: %s", err) + } else { + mysqlURI = host + } } - postgresURI = host } - { - host, err := docker.StartMySQLServer(ctx) - if err != nil { - t.Fatalf("starting mysql failed: %s", err) + } + + // Try native installation for any missing databases (Linux only) + if postgresURI == "" || mysqlURI == "" { + if err := native.Supported(); err == nil { + if postgresURI == "" { + host, err := native.StartPostgreSQLServer(ctx) + if err != nil { + t.Logf("native postgresql startup failed: %s", err) + } else { + postgresURI = host + } + } + if mysqlURI == "" { + host, err := native.StartMySQLServer(ctx) + if err != nil { + t.Logf("native mysql startup failed: %s", err) + } else { + mysqlURI = host + } } - mysqlURI = host } } + // Log which databases are available + t.Logf("PostgreSQL available: %v (URI: %s)", postgresURI != "", postgresURI) + t.Logf("MySQL available: %v (URI: %s)", mysqlURI != "", mysqlURI) + contexts := map[string]textContext{ "base": { Mutate: func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} }, @@ -138,19 +179,20 @@ func TestReplay(t *testing.T) { "managed-db": { Mutate: func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) { + // Add all servers - tests will fail if database isn't available c.Servers = []config.Server{ { Name: "postgres", Engine: config.EnginePostgreSQL, URI: postgresURI, }, - { Name: "mysql", Engine: config.EngineMySQL, URI: mysqlURI, }, } + for i := range c.SQL { switch c.SQL[i].Engine { case config.EnginePostgreSQL: @@ -172,8 +214,8 @@ func TestReplay(t *testing.T) { } }, Enabled: func() bool { - err := docker.Installed() - return err == nil + // Enabled if at least one database URI is available + return postgresURI != "" || mysqlURI != "" }, }, } diff --git a/internal/sqltest/local/mysql.go b/internal/sqltest/local/mysql.go index dedd3dfd78..05733f6e8b 100644 --- a/internal/sqltest/local/mysql.go +++ b/internal/sqltest/local/mysql.go @@ -14,6 +14,7 @@ import ( migrate "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) var mysqlSync sync.Once @@ -31,8 +32,15 @@ func MySQL(t *testing.T, migrations []string) string { t.Fatal(err) } dburi = u + } else if ierr := native.Supported(); ierr == nil { + // Fall back to native installation when Docker is not available + u, err := native.StartMySQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u } else { - t.Skip("MYSQL_SERVER_URI is empty") + t.Skip("MYSQL_SERVER_URI is empty and neither Docker nor native installation is available") } } diff --git a/internal/sqltest/local/postgres.go b/internal/sqltest/local/postgres.go index feda4cf7ac..243a7133ab 100644 --- a/internal/sqltest/local/postgres.go +++ b/internal/sqltest/local/postgres.go @@ -16,6 +16,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/pgx/poolcache" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) var flight singleflight.Group @@ -41,8 +42,15 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string { t.Fatal(err) } dburi = u + } else if ierr := native.Supported(); ierr == nil { + // Fall back to native installation when Docker is not available + u, err := native.StartPostgreSQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u } else { - t.Skip("POSTGRESQL_SERVER_URI is empty") + t.Skip("POSTGRESQL_SERVER_URI is empty and neither Docker nor native installation is available") } } diff --git a/internal/sqltest/native/enabled.go b/internal/sqltest/native/enabled.go new file mode 100644 index 0000000000..e5e12ccd80 --- /dev/null +++ b/internal/sqltest/native/enabled.go @@ -0,0 +1,20 @@ +package native + +import ( + "fmt" + "os/exec" + "runtime" +) + +// Supported returns nil if native database installation is supported on this platform. +// Currently only Linux (Ubuntu/Debian) is supported. +func Supported() error { + if runtime.GOOS != "linux" { + return fmt.Errorf("native database installation only supported on linux, got %s", runtime.GOOS) + } + // Check if apt-get is available (Debian/Ubuntu) + if _, err := exec.LookPath("apt-get"); err != nil { + return fmt.Errorf("apt-get not found: %w", err) + } + return nil +} diff --git a/internal/sqltest/native/mysql.go b/internal/sqltest/native/mysql.go new file mode 100644 index 0000000000..82881fdfb7 --- /dev/null +++ b/internal/sqltest/native/mysql.go @@ -0,0 +1,197 @@ +package native + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "os/exec" + "time" + + _ "github.com/go-sql-driver/mysql" + "golang.org/x/sync/singleflight" +) + +var mysqlFlight singleflight.Group +var mysqlURI string + +// StartMySQLServer starts an existing MySQL installation natively (without Docker). +func StartMySQLServer(ctx context.Context) (string, error) { + if err := Supported(); err != nil { + return "", err + } + if mysqlURI != "" { + return mysqlURI, nil + } + value, err, _ := mysqlFlight.Do("mysql", func() (interface{}, error) { + uri, err := startMySQLServer(ctx) + if err != nil { + return "", err + } + mysqlURI = uri + return uri, nil + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} + +func startMySQLServer(ctx context.Context) (string, error) { + // Standard URI for test MySQL + uri := "root:mysecretpassword@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true" + + // Try to connect first - it might already be running + if err := waitForMySQL(ctx, uri, 500*time.Millisecond); err == nil { + slog.Info("native/mysql", "status", "already running") + return uri, nil + } + + // Also try without password (default MySQL installation) + uriNoPassword := "root@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true" + if err := waitForMySQL(ctx, uriNoPassword, 500*time.Millisecond); err == nil { + slog.Info("native/mysql", "status", "already running (no password)") + // MySQL is running without password, try to set one + if err := setMySQLPassword(ctx); err != nil { + slog.Debug("native/mysql", "set-password-error", err) + // Return without password if we can't set one + return uriNoPassword, nil + } + // Try again with password + if err := waitForMySQL(ctx, uri, 1*time.Second); err == nil { + return uri, nil + } + // If password didn't work, use no password + return uriNoPassword, nil + } + + // Try to start existing MySQL service (might be installed but not running) + if _, err := exec.LookPath("mysqld"); err == nil { + slog.Info("native/mysql", "status", "starting existing service") + if err := startMySQLService(); err != nil { + slog.Debug("native/mysql", "start-error", err) + } else { + // Wait for MySQL to be ready + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + // Try with password first + if err := waitForMySQL(waitCtx, uri, 15*time.Second); err == nil { + return uri, nil + } + + // Try without password + if err := waitForMySQL(waitCtx, uriNoPassword, 15*time.Second); err == nil { + if err := setMySQLPassword(ctx); err != nil { + slog.Debug("native/mysql", "set-password-error", err) + return uriNoPassword, nil + } + if err := waitForMySQL(ctx, uri, 1*time.Second); err == nil { + return uri, nil + } + return uriNoPassword, nil + } + } + } + + return "", fmt.Errorf("MySQL is not installed or could not be started") +} + +func startMySQLService() error { + // Try systemctl first + cmd := exec.Command("sudo", "systemctl", "start", "mysql") + if err := cmd.Run(); err == nil { + // Give MySQL time to fully initialize + time.Sleep(2 * time.Second) + return nil + } + + // Try mysqld + cmd = exec.Command("sudo", "systemctl", "start", "mysqld") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + // Try service command + cmd = exec.Command("sudo", "service", "mysql", "start") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + cmd = exec.Command("sudo", "service", "mysqld", "start") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + return fmt.Errorf("could not start MySQL service") +} + +func setMySQLPassword(ctx context.Context) error { + // Connect without password + db, err := sql.Open("mysql", "root@tcp(localhost:3306)/mysql") + if err != nil { + return err + } + defer db.Close() + + // Set root password using mysql_native_password for broader compatibility + _, err = db.ExecContext(ctx, "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'mysecretpassword';") + if err != nil { + // Try without specifying auth plugin + _, err = db.ExecContext(ctx, "ALTER USER 'root'@'localhost' IDENTIFIED BY 'mysecretpassword';") + if err != nil { + // Try older MySQL syntax + _, err = db.ExecContext(ctx, "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('mysecretpassword');") + if err != nil { + return fmt.Errorf("could not set MySQL password: %w", err) + } + } + } + + // Flush privileges + _, _ = db.ExecContext(ctx, "FLUSH PRIVILEGES;") + + return nil +} + +func waitForMySQL(ctx context.Context, uri string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + var lastErr error + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w (last error: %v)", ctx.Err(), lastErr) + case <-ticker.C: + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for MySQL (last error: %v)", lastErr) + } + db, err := sql.Open("mysql", uri) + if err != nil { + lastErr = err + slog.Debug("native/mysql", "open-attempt", err) + continue + } + // Use a short timeout for ping to avoid hanging + pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err = db.PingContext(pingCtx) + cancel() + if err != nil { + lastErr = err + db.Close() + continue + } + db.Close() + return nil + } + } +} diff --git a/internal/sqltest/native/postgres.go b/internal/sqltest/native/postgres.go new file mode 100644 index 0000000000..f805a40a1c --- /dev/null +++ b/internal/sqltest/native/postgres.go @@ -0,0 +1,221 @@ +package native + +import ( + "context" + "fmt" + "log/slog" + "os/exec" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "golang.org/x/sync/singleflight" +) + +var postgresFlight singleflight.Group +var postgresURI string + +// StartPostgreSQLServer starts an existing PostgreSQL installation natively (without Docker). +func StartPostgreSQLServer(ctx context.Context) (string, error) { + if err := Supported(); err != nil { + return "", err + } + if postgresURI != "" { + return postgresURI, nil + } + value, err, _ := postgresFlight.Do("postgresql", func() (interface{}, error) { + uri, err := startPostgreSQLServer(ctx) + if err != nil { + return "", err + } + postgresURI = uri + return uri, nil + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} + +func startPostgreSQLServer(ctx context.Context) (string, error) { + // Standard URI for test PostgreSQL + uri := "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" + + // Try to connect first - it might already be running + if err := waitForPostgres(ctx, uri, 500*time.Millisecond); err == nil { + slog.Info("native/postgres", "status", "already running") + return uri, nil + } + + // Check if PostgreSQL is installed + if _, err := exec.LookPath("psql"); err != nil { + return "", fmt.Errorf("PostgreSQL is not installed (psql not found)") + } + + // Start PostgreSQL service + slog.Info("native/postgres", "status", "starting service") + + // Try systemctl first, fall back to pg_ctlcluster + if err := startPostgresService(); err != nil { + return "", fmt.Errorf("failed to start PostgreSQL: %w", err) + } + + // Configure PostgreSQL for password authentication + if err := configurePostgres(); err != nil { + return "", fmt.Errorf("failed to configure PostgreSQL: %w", err) + } + + // Wait for PostgreSQL to be ready + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + if err := waitForPostgres(waitCtx, uri, 30*time.Second); err != nil { + return "", fmt.Errorf("timeout waiting for PostgreSQL: %w", err) + } + + return uri, nil +} + +func startPostgresService() error { + // Try systemctl first + cmd := exec.Command("sudo", "systemctl", "start", "postgresql") + if err := cmd.Run(); err == nil { + return nil + } + + // Try service command + cmd = exec.Command("sudo", "service", "postgresql", "start") + if err := cmd.Run(); err == nil { + return nil + } + + // Try pg_ctlcluster (Debian/Ubuntu specific) + // Find the installed PostgreSQL version + output, err := exec.Command("ls", "/etc/postgresql/").CombinedOutput() + if err != nil { + return fmt.Errorf("could not find PostgreSQL version: %w", err) + } + + versions := strings.Fields(string(output)) + if len(versions) == 0 { + return fmt.Errorf("no PostgreSQL version found in /etc/postgresql/") + } + + version := versions[0] + cmd = exec.Command("sudo", "pg_ctlcluster", version, "main", "start") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("pg_ctlcluster start failed: %w\n%s", err, output) + } + + return nil +} + +func configurePostgres() error { + // Set password for postgres user using sudo -u postgres + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER USER postgres PASSWORD 'postgres';") + if output, err := cmd.CombinedOutput(); err != nil { + // This might fail if password is already set, which is fine + slog.Debug("native/postgres", "set-password", string(output)) + } + + // Update pg_hba.conf to allow password authentication + // First, find the pg_hba.conf file + output, err := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SHOW hba_file;").CombinedOutput() + if err != nil { + return fmt.Errorf("could not find hba_file: %w", err) + } + + hbaFile := strings.TrimSpace(string(output)) + if hbaFile == "" { + return fmt.Errorf("empty hba_file path") + } + + // Check if we need to update pg_hba.conf + catOutput, err := exec.Command("sudo", "cat", hbaFile).CombinedOutput() + if err != nil { + return fmt.Errorf("could not read %s: %w", hbaFile, err) + } + + // If md5 or scram-sha-256 auth is not configured for local connections, add it + content := string(catOutput) + if !strings.Contains(content, "host all all 127.0.0.1/32 md5") && + !strings.Contains(content, "host all all 127.0.0.1/32 scram-sha-256") { + + // Prepend a rule for localhost password authentication + newRule := "host all all 127.0.0.1/32 md5\n" + + // Use sed to add the rule at the beginning (after comments) + cmd := exec.Command("sudo", "bash", "-c", + fmt.Sprintf(`echo '%s' | cat - %s > /tmp/pg_hba.conf.new && sudo mv /tmp/pg_hba.conf.new %s`, + newRule, hbaFile, hbaFile)) + if output, err := cmd.CombinedOutput(); err != nil { + slog.Debug("native/postgres", "update-hba-error", string(output)) + } + + // Reload PostgreSQL to apply changes + if err := reloadPostgres(); err != nil { + slog.Debug("native/postgres", "reload-error", err) + } + } + + return nil +} + +func reloadPostgres() error { + // Try systemctl reload + cmd := exec.Command("sudo", "systemctl", "reload", "postgresql") + if err := cmd.Run(); err == nil { + return nil + } + + // Try service reload + cmd = exec.Command("sudo", "service", "postgresql", "reload") + if err := cmd.Run(); err == nil { + return nil + } + + // Try pg_ctlcluster reload + output, _ := exec.Command("ls", "/etc/postgresql/").CombinedOutput() + versions := strings.Fields(string(output)) + if len(versions) > 0 { + cmd = exec.Command("sudo", "pg_ctlcluster", versions[0], "main", "reload") + return cmd.Run() + } + + return fmt.Errorf("could not reload PostgreSQL") +} + +func waitForPostgres(ctx context.Context, uri string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + var lastErr error + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w (last error: %v)", ctx.Err(), lastErr) + case <-ticker.C: + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for PostgreSQL (last error: %v)", lastErr) + } + conn, err := pgx.Connect(ctx, uri) + if err != nil { + lastErr = err + slog.Debug("native/postgres", "connect-attempt", err) + continue + } + if err := conn.Ping(ctx); err != nil { + lastErr = err + conn.Close(ctx) + continue + } + conn.Close(ctx) + return nil + } + } +}