Support KAS

Changelog.md

@@ -3,6 +3,13 @@
 This file only reflects the changes that are made in this image. Please refer to the upstream GitLab [CHANGELOG](https://
 gitlab.com/gitlab-org/gitlab-foss/blob/master/CHANGELOG.md) for the list of changes in GitLab.
 
+## 18.7.0
+
+- gitlab: upgrade CE to v18.7.0
+- gitaly: upgrade to v18.7.0
+- gitlab-pages: upgrade to v18.7.0
+- gitlab-shell: upgrade to v14.45.5
+
 ## 18.6.2
 
 - gitlab: upgrade CE to v18.6.2

Dockerfile

@@ -1,15 +1,16 @@
 FROM ubuntu:noble-20251013
 
-ARG VERSION=18.6.2
+ARG VERSION=18.7.0
 
 ENV GITLAB_VERSION=${VERSION} \
     RUBY_VERSION=3.2.9 \
     RUBY_SOURCE_SHA256SUM="abbad98db9aeb152773b0d35868e50003b8c467f3d06152577c4dfed9d88ed2a" \
     RUBYGEMS_VERSION=3.7.2 \
     GOLANG_VERSION=1.24.11 \
-    GITLAB_SHELL_VERSION=14.45.3 \
-    GITLAB_PAGES_VERSION=18.6.2 \
-    GITALY_SERVER_VERSION=18.6.2 \
+    GITLAB_SHELL_VERSION=14.45.5 \
+    GITLAB_PAGES_VERSION=18.7.0 \
+    GITALY_SERVER_VERSION=18.7.0 \
+    GITLAB_AGENT_VERSION=18.7.0 \
     GITLAB_USER="git" \
     GITLAB_HOME="/home/git" \
     GITLAB_LOG_DIR="/var/log/gitlab" \
@@ -21,6 +22,7 @@ ENV GITLAB_VERSION=${VERSION} \
 ENV GITLAB_INSTALL_DIR="${GITLAB_HOME}/gitlab" \
     GITLAB_SHELL_INSTALL_DIR="${GITLAB_HOME}/gitlab-shell" \
     GITLAB_GITALY_INSTALL_DIR="${GITLAB_HOME}/gitaly" \
+    GITLAB_AGENT_INSTALL_DIR="${GITLAB_HOME}/gitlab-agent" \
     GITLAB_DATA_DIR="${GITLAB_HOME}/data" \
     GITLAB_BUILD_DIR="${GITLAB_CACHE_DIR}/build" \
     GITLAB_RUNTIME_DIR="${GITLAB_CACHE_DIR}/runtime"

VERSION

@@ -1 +1 @@
-18.6.2
+18.7.0

assets/build/install.sh

@@ -5,10 +5,12 @@ GITLAB_CLONE_URL=https://gitlab.com/gitlab-org/gitlab-foss.git
 GITLAB_SHELL_URL=https://gitlab.com/gitlab-org/gitlab-shell/-/archive/v${GITLAB_SHELL_VERSION}/gitlab-shell-v${GITLAB_SHELL_VERSION}.tar.bz2
 GITLAB_PAGES_URL=https://gitlab.com/gitlab-org/gitlab-pages.git
 GITLAB_GITALY_URL=https://gitlab.com/gitlab-org/gitaly.git
+GITLAB_AGENT_URL=https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent.git
 
 GITLAB_WORKHORSE_BUILD_DIR=${GITLAB_INSTALL_DIR}/workhorse
 GITLAB_PAGES_BUILD_DIR=/tmp/gitlab-pages
 GITLAB_GITALY_BUILD_DIR=/tmp/gitaly
+GITLAB_AGENT_BUILD_DIR=/tmp/gitlab-agent
 
 RUBY_SRC_URL=https://cache.ruby-lang.org/pub/ruby/${RUBY_VERSION%.*}/ruby-${RUBY_VERSION}.tar.gz
 
@@ -171,6 +173,18 @@ make -C ${GITLAB_GITALY_BUILD_DIR} git GIT_PREFIX=/usr/local
 # clean up
 rm -rf ${GITLAB_GITALY_BUILD_DIR}
 
+# download gitlab-agent (KAS)
+echo "Downloading gitlab-agent v.${GITLAB_AGENT_VERSION}..."
+git clone -q -b v${GITLAB_AGENT_VERSION} --depth 1 ${GITLAB_AGENT_URL} ${GITLAB_AGENT_BUILD_DIR}
+
+# install gitlab-agent (KAS)
+mkdir -p "${GITLAB_AGENT_INSTALL_DIR}"
+make -C ${GITLAB_AGENT_BUILD_DIR} kas TARGET_DIRECTORY=/usr/local/bin
+chown -R ${GITLAB_USER}: ${GITLAB_AGENT_INSTALL_DIR}
+
+# clean up
+rm -rf ${GITLAB_AGENT_BUILD_DIR}
+
 # remove go
 go clean --modcache
 rm -rf ${GITLAB_BUILD_DIR}/go${GOLANG_VERSION}.linux-amd64.tar.gz ${GOROOT}
@@ -411,6 +425,20 @@ stdout_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
 stderr_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
 EOF
 
+# configure superisord to start gitlab-agent (KAS)
+cat > /etc/supervisor/conf.d/gitlab-kas.conf <<EOF
+[program:gitlab_kas]
+priority=5
+directory=${GITLAB_AGENT_INSTALL_DIR}
+environment=HOME=${GITLAB_HOME}
+command=/usr/local/bin/kas --configuration-file="${GITLAB_AGENT_INSTALL_DIR}/gitlab-kas_config.yaml"
+user=git
+autostart={{GITLAB_AGENT_BUILTIN_KAS_ENABLED}}
+autorestart=true
+stdout_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
+stderr_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
+EOF
+
 # configure supervisord to start mail_room
 cat > /etc/supervisor/conf.d/mail_room.conf <<EOF
 [program:mail_room]

assets/runtime/config/gitlab-agent/gitlab-kas_config.yaml

@@ -0,0 +1,95 @@
+# This is a configuration file for kas that contains the default values for the settings.
+# It DOES NOT contain all the possible configuration knobs.
+# The source of truth is kascfg.proto.
+# It contains all the fields and documentation them.
+# If you are looking for a setting, start from the ConfigurationFile message in:
+# - the proto file kascfg.proto.
+# - the generated documentation in kascfg_proto_docs.md.
+# Correctness of this file is enforced by a unit test in kascfg_defaults_test.go.
+
+agent:
+  configuration:
+    max_configuration_file_size: 131072
+    poll_period: 300s
+  info_cache_error_ttl: 60s
+  info_cache_ttl: 300s
+  kubernetes_api:
+    allowed_agent_cache_error_ttl: 10s
+    allowed_agent_cache_ttl: 60s
+    listen:
+      address: 127.0.0.1:8154
+      listen_grace_period: 5s
+      network: tcp
+      shutdown_grace_period: 3600s
+    url_path_prefix: {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/k8s-proxy/
+    websocket_token_secret_file: {{GITLAB_AGENT_KAS_WEBSOCKET_TOKEN_SECRET_FILE}}
+  listen:
+    address: 127.0.0.1:8150
+    connections_per_token_per_minute: 40000
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    websocket: true
+  receptive_agent:
+    poll_period: 60s
+  redis_conn_info_gc: 600s
+  redis_conn_info_refresh: 240s
+  redis_conn_info_ttl: 300s
+api:
+  listen:
+    address: 127.0.0.1:8153
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    authentication_secret_file: {{GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE}}  # required
+gitaly:
+  global_api_rate_limit:
+    bucket_size: 70
+    refill_rate_per_second: 30
+  per_server_api_rate_limit:
+    bucket_size: 40
+    refill_rate_per_second: 15
+gitlab:
+  address: http://127.0.0.1:8080{{GITLAB_RELATIVE_URL_ROOT}}
+  authentication_secret_file: {{GITLAB_AGENT_KAS_GITLAB_AUTHENTICATION_SECRET_FILE}} # required
+  api_rate_limit:
+    bucket_size: 250
+    refill_rate_per_second: 50
+observability:
+  event_reporting_period: 300s
+  google_profiler: {}
+  listen:
+    address: 127.0.0.1:8151
+    network: tcp
+  liveness_probe:
+    url_path: /liveness
+  logging:
+    level: debug
+    grpc_level: debug
+  prometheus:
+    url_path: /metrics
+  readiness_probe:
+    url_path: /readiness
+  sentry: {}
+  usage_reporting_period: 10s
+private_api:
+  listen:
+    address: 0.0.0.0:8155
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    authentication_secret_file: {{GITLAB_AGENT_KAS_PRIVATE_API_LISTEN_AUTHENTICATION_SECRET_FILE}}
+redis:
+  dial_timeout: 5s
+  key_prefix: gitlab-kas
+  network: tcp
+  write_timeout: 3s
+  server:
+    address: "{{REDIS_HOST}}:{{REDIS_PORT}}" # required
+workspaces:
+  listen:
+    address: 127.0.0.1:8160
+    listen_grace_period: 5s
+    network: tcp
+    shutdown_grace_period: 3600s
+

assets/runtime/config/gitlabhq/gitlab.yml

@@ -1174,6 +1174,22 @@ production: &base
     # Default is '.gitlab_workhorse_secret' relative to Rails.root (i.e. root of the GitLab app).
     # secret_file: /home/git/gitlab/.gitlab_workhorse_secret
 
+  gitlab_kas:
+    enabled: {{GITLAB_KAS_ENABLED}}
+    # File that contains the secret key for verifying access for gitlab-kas.
+    # Default is '.gitlab_kas_secret' relative to Rails.root (i.e. root of the GitLab app).
+    secret_file: {{GITLAB_KAS_SECRET}} # /home/git/gitlab/.gitlab_kas_secret
+
+    # The URL to the external KAS API (used by the Kubernetes agents)
+    external_url: {{GITLAB_KAS_EXTERNAL}} # wss://kas.example.com
+
+    # The URL to the internal KAS API (used by the GitLab backend)
+    internal_url: {{GITLAB_KAS_INTERNAL}} # grpc://localhost:8153
+
+    # The URL to the Kubernetes API proxy (used by GitLab users)
+    external_k8s_proxy_url: {{GITLAB_KAS_PROXY}} # https://localhost:8154 # default: nil
+
+
   ## GitLab Elasticsearch settings
   elasticsearch:
     indexer_path: {{GITLAB_HOME}}/gitlab-elasticsearch-indexer/
@@ -1358,7 +1374,7 @@ test:
         region: us-east-1
 
   gitlab:
-    host: localhost
+    host: 127.0.0.1
     port: 80
 
     content_security_policy:

assets/runtime/config/gitlabhq/puma.rb

@@ -34,34 +34,50 @@
 # Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
 # accepted protocols.
 bind 'unix:///home/git/gitlab/tmp/sockets/gitlab.socket'
-bind 'tcp://127.0.0.1:8080'
 
 workers {{PUMA_WORKERS}}
 
 require_relative "{{GITLAB_INSTALL_DIR}}/lib/gitlab/cluster/lifecycle_events"
 
-on_restart do
-  # Signal application hooks that we're about to restart
-  Gitlab::Cluster::LifecycleEvents.do_before_master_restart
-end
+if Gem::Version.new(Puma::Const::PUMA_VERSION) < Gem::Version.new('7.0')
+  Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
 
-before_fork do
-  # Signal to the puma killer
-  Gitlab::Cluster::PumaWorkerKillerInitializer.start(@config.options, puma_per_worker_max_memory_mb: {{PUMA_PER_WORKER_MAX_MEMORY_MB}}, puma_master_max_memory_mb: {{PUMA_MASTER_MAX_MEMORY_MB}}) unless ENV['DISABLE_PUMA_WORKER_KILLER']
+  on_restart do
+    # Signal application hooks that we're about to restart
+    Gitlab::Cluster::LifecycleEvents.do_before_master_restart
+  end
 
-  # Signal application hooks that we're about to fork
-  Gitlab::Cluster::LifecycleEvents.do_before_fork
-end
+  on_worker_boot do
+    # Signal application hooks of worker start
+    Gitlab::Cluster::LifecycleEvents.do_worker_start
+  end
 
-Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
-on_worker_boot do
-  # Signal application hooks of worker start
-  Gitlab::Cluster::LifecycleEvents.do_worker_start
+  on_worker_shutdown do
+    # Signal application hooks that a worker is shutting down
+    Gitlab::Cluster::LifecycleEvents.do_worker_stop
+  end
+else
+  Gitlab::Cluster::LifecycleEvents.set_puma_worker_count(3)
+
+  before_restart do
+    # Signal application hooks that we're about to restart
+    Gitlab::Cluster::LifecycleEvents.do_before_master_restart
+  end
+
+  before_worker_boot do
+    # Signal application hooks of worker start
+    Gitlab::Cluster::LifecycleEvents.do_worker_start
+  end
+
+  before_worker_shutdown do
+    # Signal application hooks that a worker is shutting down
+    Gitlab::Cluster::LifecycleEvents.do_worker_stop
+  end
 end
 
-on_worker_shutdown do
-  # Signal application hooks that a worker is shutting down
-  Gitlab::Cluster::LifecycleEvents.do_worker_stop
+before_fork do
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
 end
 
 # Preload the application before starting the workers; this conflicts with
@@ -87,11 +103,10 @@
   json_formatter.call(str)
 end
 
-lowlevel_error_handler do |ex, env|
-  if Raven.configuration.capture_allowed?
-    Raven.capture_exception(ex, tags: { 'handler': 'puma_low_level' }, extra: { puma_env: env })
-  end
+require_relative "{{GITLAB_INSTALL_DIR}}/lib/gitlab/puma/error_handler"
+
+error_handler = Gitlab::Puma::ErrorHandler.new(ENV['RAILS_ENV'] == 'production')
 
-  # note the below is just a Rack response
-  [500, {}, ["An error has occurred and reported in the system's low-level error handler."]]
+lowlevel_error_handler do |ex, env, status_code|
+  error_handler.execute(ex, env, status_code)
 end

assets/runtime/config/nginx/gitlab

@@ -84,6 +84,54 @@ server {
     proxy_pass http://gitlab-workhorse;
   }
 
+  #start-builtin-kas
+  location {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/ {
+    client_max_body_size 0;
+    gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+    proxy_buffering         {{NGINX_PROXY_BUFFERING}};
+
+    proxy_http_version 1.1;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{NGINX_X_FORWARDED_PROTO}};
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
+
+    proxy_pass http://127.0.0.1:8150;
+  }
+
+  location {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/k8s-proxy/ {
+    client_max_body_size 0;
+    gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+    proxy_buffering         {{NGINX_PROXY_BUFFERING}};
+
+    proxy_http_version 1.1;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{NGINX_X_FORWARDED_PROTO}};
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
+
+    proxy_pass http://127.0.0.1:8154;
+  }
+  #end-builtin-kas
+
   error_page 404 /404.html;
   error_page 422 /422.html;
   error_page 500 /500.html;