diff --git a/.github/workflows/drift_deploy.yml b/.github/workflows/drift_deploy.yml deleted file mode 100644 index b42591950..000000000 --- a/.github/workflows/drift_deploy.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: driftapp Deploy -on: - push: - branches: - - develop # change to main if needed - - feat-drift-app -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only --config fly-drift.toml - env: - FLY_API_TOKEN: ${{ secrets.FLYIO_DRIFT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/pro-deploy.yml b/.github/workflows/pro-deploy.yml index 6f37d4212..4545f1457 100644 --- a/.github/workflows/pro-deploy.yml +++ b/.github/workflows/pro-deploy.yml @@ -1,19 +1,19 @@ # See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ -name: Deploy pro-backend +name: Deploy projects-refresh-service on: push: branches: - develop - - pro jobs: deploy: - name: Deploy app + name: Deploy projects-refresh-service app runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time + concurrency: projects-refresh # optional: ensure only one action runs at a time steps: - uses: actions/checkout@v4 - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only --config fly-pro.toml + - run: flyctl deploy --remote-only --config fly-projects-refresh-service.toml --image-label latest + env: - FLY_API_TOKEN: ${{ secrets.FLYIO_PRO_TOKEN }} + FLY_API_TOKEN: ${{ secrets.FLYIO_PROJECTS_REFRESH_SVC_TOKEN }} diff --git a/Dockerfile_bg_projects_refresh b/Dockerfile_bg_projects_refresh new file mode 100644 index 000000000..02356e654 --- /dev/null +++ b/Dockerfile_bg_projects_refresh @@ -0,0 +1,28 @@ +FROM golang:1.24.0 as builder +ARG COMMIT_SHA +RUN echo "commit sha: ${COMMIT_SHA}" + +# Set the working directory +WORKDIR $GOPATH/src/github.com/diggerhq/digger + +# Copy all required source, blacklist files that are not required through `.dockerignore` +COPY . . + +RUN go build -ldflags="-X 'main.Version=${COMMIT_SHA}'" -o projects_refresh_exe ./background/projects-refresh-service + +# Multi-stage build will just copy the binary to an alpine image. +FROM ubuntu:24.04 as runner +ARG COMMIT_SHA +WORKDIR /app + +RUN apt-get update && apt-get install -y ca-certificates curl && apt-get install -y git && apt-get clean all +RUN update-ca-certificates + +RUN echo "commit sha: ${COMMIT_SHA}" + +# Copy the binary to the corresponding folder +COPY --from=builder /go/src/github.com/diggerhq/digger/projects_refresh_exe /app/projects_refresh_exe +RUN chmod +x projects_refresh_exe + +# Run the binary +ENTRYPOINT ["./projects_refresh_exe"] diff --git a/backend/controllers/cache.go b/backend/controllers/cache.go index 86f04dce7..8f519fd03 100644 --- a/backend/controllers/cache.go +++ b/backend/controllers/cache.go @@ -2,6 +2,7 @@ package controllers import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "net/http" "os" @@ -67,7 +68,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) { // update the cache here, do it async for immediate response go func() { - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 539eeb136..95b9d34b9 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "math/rand" "net/http" @@ -915,7 +916,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 var diggerYmlStr string var dependencyGraph graph.Graph[string, dg_configuration.Project] - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { slog.Debug("Reading Digger config from cloned repository", "directory", dir) diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir) diff --git a/backend/models/storage.go b/backend/models/storage.go index 4acfb28a5..3cc1e0c4a 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -1642,6 +1642,7 @@ func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.D err = db.GormDB.Transaction(func(tx *gorm.DB) error { for _, dc := range config.Projects { + slog.Debug("refreshing for project", "name", dc.Name, "dir", dc.Dir) projectName := dc.Name projectDirectory := dc.Dir p, err := db.GetProjectByName(orgId, repoFullName, projectName) diff --git a/backend/service_clients/projects_service.go b/backend/service_clients/projects_service.go new file mode 100644 index 000000000..cec8c7719 --- /dev/null +++ b/backend/service_clients/projects_service.go @@ -0,0 +1,107 @@ +package service_clients + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "time" +) + +type MachineConfig struct { + Name string `json:"name"` + Config struct { + Image string `json:"image"` + Env struct { + CloneUrl string `json:"DIGGER_GITHUB_REPO_CLONE_URL"` + Branch string `json:"DIGGER_GITHUB_REPO_CLONE_BRANCH"` + GithubToken string `json:"DIGGER_GITHUB_TOKEN"` + RepoFullName string `json:"DIGGER_REPO_FULL_NAME"` + OrgId string `json:"DIGGER_ORG_ID"` + } `json:"env"` + Guest struct { + CPUs int `json:"cpus"` + CPUKind string `json:"cpu_kind"` + MemoryMB int `json:"memory_mb"` + } `json:"guest"` + AutoDestroy bool `json:"auto_destroy"` + } `json:"config"` +} + +type MachineResponse struct { + ID string `json:"id"` +} + +type QueuedResponse struct { + Queued string `json:"queued"` +} + +func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken string, repoFullName string, orgId string) (*MachineResponse, error) { + + slog.Debug("awaiting machine fetch") + + // Prepare machine configuration + machineConfig := MachineConfig{ + Name: fmt.Sprintf("hello-%d", time.Now().UnixMilli()), + } + + machineConfig.Config.Image = "registry.fly.io/projects-refresh-service:latest" + machineConfig.Config.Env.CloneUrl = cloneUrl + machineConfig.Config.Env.Branch = branch + machineConfig.Config.Env.GithubToken = githubToken + machineConfig.Config.Env.RepoFullName = repoFullName + machineConfig.Config.Env.OrgId = orgId + + machineConfig.Config.Guest.CPUs = 1 + machineConfig.Config.Guest.CPUKind = "shared" + machineConfig.Config.Guest.MemoryMB = 256 + machineConfig.Config.AutoDestroy = true + + // Marshal JSON payload + payload, err := json.Marshal(machineConfig) + if err != nil { + slog.Error("Error creating machine config", "error", err) + return nil, err + } + + // Create HTTP request + apiURL := fmt.Sprintf("https://api.machines.dev/v1/apps/%s/machines", os.Getenv("DIGGER_PROJECTS_SVC_APP_NAME")) + req2, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload)) + if err != nil { + slog.Error("Error creating request", "error", err) + return nil, err + } + + // Set headers + req2.Header.Set("Authorization", "Bearer "+os.Getenv("FLY_PROJECTS_SVC_API_TOKEN")) + req2.Header.Set("Content-Type", "application/json") + + // Make HTTP request + client := &http.Client{} + resp, err := client.Do(req2) + if err != nil { + slog.Error("Error making request", "error", err) + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, err2 := io.ReadAll(resp.Body) + slog.Error("Error triggering projects refresh service", "statusCode", resp.StatusCode, "body", body, "readyErr", err2) + return nil, fmt.Errorf("error triggering projects refresh service") + } + + // Parse response + var machineResp MachineResponse + if err := json.NewDecoder(resp.Body).Decode(&machineResp); err != nil { + slog.Error("Error parsing response", "error", err) + return nil, err + } + + slog.Debug("triggered projects refresh service", "machineId", machineResp.ID, "statusCode", resp.StatusCode) + + return &MachineResponse{ID: machineResp.ID}, nil +} diff --git a/backend/services/repos.go b/backend/services/repos.go index 1efb9a916..dce68940f 100644 --- a/backend/services/repos.go +++ b/backend/services/repos.go @@ -3,8 +3,8 @@ package services import ( "fmt" "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/service_clients" utils3 "github.com/diggerhq/digger/backend/utils" - dg_configuration "github.com/diggerhq/digger/libs/digger_config" "log/slog" "strconv" "strings" @@ -41,17 +41,12 @@ func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId s return fmt.Errorf("error getting github service") } - err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { - config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) - if err != nil { - slog.Error("failed to load digger.yml: %v", "error", err) - return fmt.Errorf("error loading digger.yml %v", err) - } - models.DB.RefreshProjectsFromRepo(strconv.Itoa(int(link.OrganisationId)), *config, repoFullName) - return nil - }) + resp, err := service_clients.TriggerProjectsRefreshService(cloneUrl, branch, *token, repoFullName, strconv.Itoa(int(link.OrganisationId))) if err != nil { - return fmt.Errorf("error while cloning repo: %v", err) + return fmt.Errorf("error triggering projects refresh service: %v", err) + } + if resp == nil { + return fmt.Errorf("error triggering projects refresh service: response nil") } return nil diff --git a/backend/utils/bitbucket.go b/backend/utils/bitbucket.go index c47468ae0..7a14ab5ce 100644 --- a/backend/utils/bitbucket.go +++ b/backend/utils/bitbucket.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "net/http" "os" @@ -94,7 +95,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF "changedFilesCount", len(changedFiles), ) - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error { diggerYmlPath := path.Join(dir, "digger.yml") diggerYmlBytes, err := os.ReadFile(diggerYmlPath) if err != nil { diff --git a/backend/utils/github.go b/backend/utils/github.go index d35e4821f..c45f7a568 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -18,66 +18,6 @@ import ( "github.com/google/go-github/v61/github" ) -func createTempDir() string { - tempDir, err := os.MkdirTemp("", "repo") - if err != nil { - slog.Error("Failed to create temporary directory", "error", err) - panic(err) - } - return tempDir -} - -type action func(string) error - -func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { - dir := createTempDir() - - slog.Debug("Cloning git repository", - "repoUrl", repoUrl, - "branch", branch, - "commitHash", commitHash, - "directory", dir, - ) - - git := NewGitShellWithTokenAuth(dir, token, tokenUsername) - err := git.Clone(repoUrl, branch) - if err != nil { - slog.Error("Failed to clone repository", - "repoUrl", repoUrl, - "branch", branch, - "error", err, - ) - return err - } - - if commitHash != "" { - err := git.Checkout(commitHash) - if err != nil { - slog.Error("Failed to checkout commit", - "commitHash", commitHash, - "error", err, - ) - return err - } - } - - defer func() { - slog.Debug("Removing cloned directory", "directory", dir) - ferr := os.RemoveAll(dir) - if ferr != nil { - slog.Warn("Failed to remove directory", "directory", dir, "error", ferr) - } - }() - - err = action(dir) - if err != nil { - slog.Error("Error performing action on repository", "directory", dir, "error", err) - return err - } - - return nil -} - // just a wrapper around github client to be able to use mocks type DiggerGithubRealClientProvider struct { } diff --git a/backend/utils/github_test.go b/backend/utils/github_test.go index def6d613e..51c0510c8 100644 --- a/backend/utils/github_test.go +++ b/backend/utils/github_test.go @@ -1,6 +1,7 @@ package utils import ( + "github.com/diggerhq/digger/libs/git_utils" "os" "testing" @@ -9,14 +10,14 @@ import ( func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) { f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", "", f) assert.NotNil(t, err) } func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, "", f) assert.Nil(t, err) } @@ -27,13 +28,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) { return } f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, "", f) assert.Nil(t, err) } func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, "", f) assert.NotNil(t, err) } diff --git a/backend/utils/gitlab.go b/backend/utils/gitlab.go index 187ff8e93..78c9b8aff 100644 --- a/backend/utils/gitlab.go +++ b/backend/utils/gitlab.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "os" "path" @@ -113,7 +114,7 @@ func GetDiggerConfigForBranchGitlab(gh GitlabProvider, projectId int, repoFullNa "changedFilesCount", len(changedFiles), ) - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { diggerYmlPath := path.Join(dir, "digger.yml") diggerYmlBytes, err := os.ReadFile(diggerYmlPath) if err != nil { diff --git a/background/projects-refresh-service/.gitignore b/background/projects-refresh-service/.gitignore new file mode 100644 index 000000000..88d050b19 --- /dev/null +++ b/background/projects-refresh-service/.gitignore @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/background/projects-refresh-service/go.mod b/background/projects-refresh-service/go.mod new file mode 100644 index 000000000..f743f6e4d --- /dev/null +++ b/background/projects-refresh-service/go.mod @@ -0,0 +1,3 @@ +module github.com/diggerhq/digger/background/projects-refresh-service + +go 1.24.0 diff --git a/background/projects-refresh-service/projects_refesh_main.go b/background/projects-refresh-service/projects_refesh_main.go new file mode 100644 index 000000000..0623237c5 --- /dev/null +++ b/background/projects-refresh-service/projects_refesh_main.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "github.com/diggerhq/digger/backend/models" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" + "log/slog" + "os" +) + +func init() { + logLevel := os.Getenv("DIGGER_LOG_LEVEL") + var level slog.Leveler + + if logLevel == "DEBUG" { + level = slog.LevelDebug + } else if logLevel == "WARN" { + level = slog.LevelWarn + } else if logLevel == "ERROR" { + level = slog.LevelError + } else { + level = slog.LevelInfo + } + + handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + }) + logger := slog.New(handler) + slog.SetDefault(logger) +} + +func main() { + cloneUrl := os.Getenv("DIGGER_GITHUB_REPO_CLONE_URL") + branch := os.Getenv("DIGGER_GITHUB_REPO_CLONE_BRANCH") + token := os.Getenv("DIGGER_GITHUB_TOKEN") + repoFullName := os.Getenv("DIGGER_REPO_FULL_NAME") + orgId := os.Getenv("DIGGER_ORG_ID") + + if cloneUrl == "" && branch == "" && token == "" && orgId == "" && repoFullName == "" { + slog.Info("smoketests mode, skipping this run") + os.Exit(0) + } + + models.ConnectDatabase() + + slog.Info("refreshing projects from repo", "repoFullName", repoFullName) + err := utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) + if err != nil { + slog.Error("failed to load digger.yml: %v", "error", err) + return fmt.Errorf("error loading digger.yml %v", err) + } + slog.Debug("RefreshProjectsFromRepo", "repoFullName", repoFullName) + err = models.DB.RefreshProjectsFromRepo(orgId, *config, repoFullName) + slog.Debug("finished RefreshProjectsFromRepo", "repoFullName", repoFullName, "error", err) + if err != nil { + slog.Error("failed to refresh projects from repo: %v", "error", err) + return fmt.Errorf("error refreshing projects from repo %v", err) + } + return nil + }) + if err != nil { + slog.Error("error while cloning repo: %v", err) + os.Exit(1) + } + +} diff --git a/ee/cli/pkg/policy/policy.go b/ee/cli/pkg/policy/policy.go index c53983a5d..3919fcdef 100644 --- a/ee/cli/pkg/policy/policy.go +++ b/ee/cli/pkg/policy/policy.go @@ -1,7 +1,7 @@ package policy import ( - "github.com/diggerhq/digger/ee/cli/pkg/utils" + "github.com/diggerhq/digger/libs/git_utils" "github.com/samber/lo" "os" "path" @@ -62,7 +62,7 @@ func GetPrefixesForPath(path string, fileName string) []string { func (p DiggerRepoPolicyProvider) getPolicyFileContents(repo string, projectName string, projectDir string, fileName string) (string, error) { var contents string - err := utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", p.GitToken, "", func(basePath string) error { + err := git_utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", "", p.GitToken, "", func(basePath string) error { // we start with the project directory path prefixes as the highest priority prefixes := GetPrefixesForPath(path.Join(basePath, projectDir), fileName) diff --git a/ee/cli/pkg/utils/github.go b/ee/cli/pkg/utils/github.go deleted file mode 100644 index 3655ac6f3..000000000 --- a/ee/cli/pkg/utils/github.go +++ /dev/null @@ -1,35 +0,0 @@ -package utils - -import ( - "github.com/diggerhq/digger/backend/utils" - "log" - "os" -) - -func createTempDir() string { - tempDir, err := os.MkdirTemp("", "repo") - if err != nil { - log.Fatal(err) - } - return tempDir -} - -type action func(string) error - -func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, tokenUsername string, action action) error { - dir := createTempDir() - git := utils.NewGitShellWithTokenAuth(dir, token, tokenUsername) - err := git.Clone(repoUrl, branch) - if err != nil { - return err - } - defer os.RemoveAll(dir) - err = action(dir) - if err != nil { - log.Printf("error performing action: %v", err) - return err - } - - return nil - -} diff --git a/ee/drift/tasks/github.go b/ee/drift/tasks/github.go index f7e6af4b9..ba9995555 100644 --- a/ee/drift/tasks/github.go +++ b/ee/drift/tasks/github.go @@ -2,10 +2,10 @@ package tasks import ( "fmt" - utils3 "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/ee/drift/dbmodels" "github.com/diggerhq/digger/ee/drift/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" utils2 "github.com/diggerhq/digger/next/utils" "log" "strconv" diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go index 14441ad19..0dee042e5 100644 --- a/ee/drift/utils/github.go +++ b/ee/drift/utils/github.go @@ -5,10 +5,10 @@ import ( "encoding/base64" "fmt" "github.com/bradleyfalzon/ghinstallation/v2" - utils2 "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/ee/drift/dbmodels" github2 "github.com/diggerhq/digger/libs/ci/github" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils2 "github.com/diggerhq/digger/libs/git_utils" "github.com/diggerhq/digger/next/utils" "github.com/dominikbraun/graph" "github.com/google/go-github/v61/github" diff --git a/fly-pro.toml b/fly-pro.toml index 132608001..df2f375ca 100644 --- a/fly-pro.toml +++ b/fly-pro.toml @@ -10,6 +10,9 @@ primary_region = 'lhr' HOSTNAME = 'https://ui-backend.digger.dev' DIGGER_HOSTNAME = 'https://ui-backend.digger.dev' JWT_AUTH = 'true' + DIGGER_PROJECTS_SVC_APP_NAME = 'projects-refresh-service' + DIGGER_LOAD_PROJECTS_ON_PUSH = 'true' + [build] dockerfile = 'Dockerfile_backend_ee' diff --git a/fly-projects-refresh-service.toml b/fly-projects-refresh-service.toml new file mode 100644 index 000000000..c101319c4 --- /dev/null +++ b/fly-projects-refresh-service.toml @@ -0,0 +1,23 @@ +# fly.toml app configuration file generated for projects-refresh-service on 2025-07-05T13:54:02-07:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'projects-refresh-service' +primary_region = 'lhr' + +[build] + dockerfile = 'Dockerfile_bg_projects_refresh' + +[http_service] + internal_port = 8080 + force_https = true + auto_stop_machines = 'stop' + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[vm]] + memory = '1gb' + cpu_kind = 'shared' + cpus = 1 diff --git a/go.work b/go.work index c4da94096..4122be1a9 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,7 @@ go 1.24.0 use ( ./backend + ./background/projects-refresh-service ./cli ./cli_e2e ./dgctl diff --git a/backend/utils/gitshell.go b/libs/git_utils/clone_utils.go similarity index 68% rename from backend/utils/gitshell.go rename to libs/git_utils/clone_utils.go index 9abcca0b6..5bd8b3cc8 100644 --- a/backend/utils/gitshell.go +++ b/libs/git_utils/clone_utils.go @@ -1,9 +1,10 @@ -package utils +package git_utils import ( "bytes" "context" "fmt" + "log/slog" "net/url" "os" "os/exec" @@ -11,6 +12,70 @@ import ( "time" ) +func createTempDir() (string, error) { + tempDir, err := os.MkdirTemp("", "repo") + if err != nil { + slog.Error("Failed to create temporary directory", "error", err) + return "", err + } + return tempDir, nil +} + +type action func(string) error + +func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { + dir, err := createTempDir() + if err != nil { + slog.Error("Failed to create temporary directory", "error", err) + return err + } + + slog.Debug("Cloning git repository", + "repoUrl", repoUrl, + "branch", branch, + "commitHash", commitHash, + "directory", dir, + ) + + git := NewGitShellWithTokenAuth(dir, token, tokenUsername) + err = git.Clone(repoUrl, branch) + if err != nil { + slog.Error("Failed to clone repository", + "repoUrl", repoUrl, + "branch", branch, + "error", err, + ) + return err + } + + if commitHash != "" { + err := git.Checkout(commitHash) + if err != nil { + slog.Error("Failed to checkout commit", + "commitHash", commitHash, + "error", err, + ) + return err + } + } + + defer func() { + slog.Debug("Removing cloned directory", "directory", dir) + ferr := os.RemoveAll(dir) + if ferr != nil { + slog.Warn("Failed to remove directory", "directory", dir, "error", ferr) + } + }() + + err = action(dir) + if err != nil { + slog.Error("Error performing action on repository", "directory", dir, "error", err) + return err + } + + return nil +} + type GitAuth struct { Username string Password string // Can be either password or access token diff --git a/next/controllers/github.go b/next/controllers/github.go index b14e84bb2..ecbd39e92 100644 --- a/next/controllers/github.go +++ b/next/controllers/github.go @@ -12,6 +12,7 @@ import ( "github.com/diggerhq/digger/libs/ci" dg_github "github.com/diggerhq/digger/libs/ci/github" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/git_utils" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/next/ci_backends" "github.com/diggerhq/digger/next/dbmodels" @@ -538,7 +539,7 @@ func getDiggerConfigForBranch(gh next_utils.GithubClientProvider, installationId log.Printf("Error getting changed files: %v", err) return "", nil, nil, nil, fmt.Errorf("error getting changed files") } - err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) diff --git a/next/services/config.go b/next/services/config.go index f00eee483..29d79af04 100644 --- a/next/services/config.go +++ b/next/services/config.go @@ -2,8 +2,8 @@ package services import ( "fmt" - utils3 "github.com/diggerhq/digger/backend/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" "github.com/diggerhq/digger/next/dbmodels" "github.com/diggerhq/digger/next/utils" "log"