Skip to content

Commit d66db7c

Browse files
committed
feat: new e2e uses bastion hosts for ssh
1 parent e493603 commit d66db7c

28 files changed

+1201
-1037
lines changed

.pipelines/e2e-gpu.yaml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: $(Date:yyyyMMdd)$(Rev:.r)
2+
variables:
3+
TAGS_TO_RUN: "gpu=true"
4+
TAGS_TO_SKIP: "os=windows"
5+
SKIP_E2E_TESTS: false
6+
trigger:
7+
branches:
8+
include:
9+
- main
10+
pr:
11+
branches:
12+
include:
13+
- official/*
14+
- windows/*
15+
- main
16+
paths:
17+
include:
18+
- .pipelines/e2e.yaml
19+
- .pipelines/templates/e2e-template.yaml
20+
- .pipelines/scripts/e2e_run.sh
21+
- e2e
22+
- parts/linux
23+
- parts/common/components.json
24+
- pkg/agent
25+
- go.mod
26+
- go.sum
27+
exclude:
28+
- pkg/agent/datamodel/sig_config*.go # SIG config changes
29+
- pkg/agent/datamodel/*.json # SIG version changes
30+
- pkg/agent/testdata/AKSWindows* # Windows test data
31+
- parts/common/components.json # centralized components management file
32+
- staging/cse/windows/README
33+
- /**/*.md
34+
- e2e/scenario_win_test.go
35+
36+
jobs:
37+
- template: ./templates/e2e-template.yaml
38+
parameters:
39+
name: Linux GPU Tests
40+
IgnoreScenariosWithMissingVhd: false

.pipelines/e2e.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: $(Date:yyyyMMdd)$(Rev:.r)
22
variables:
3-
TAGS_TO_SKIP: "os=windows"
3+
TAGS_TO_SKIP: "os=windows,gpu=true"
44
SKIP_E2E_TESTS: false
55
trigger:
66
branches:

.pipelines/scripts/e2e_run.sh

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,37 @@ if [ -n "${SIG_GALLERY_NAME}" ]; then
6464
export GALLERY_NAME=$SIG_GALLERY_NAME
6565
fi
6666

67+
az extension add --name bastion
68+
6769
# this software is used to take the output of "go test" and produce a junit report that we can upload to the pipeline
6870
# and see fancy test results.
6971
cd e2e
7072
mkdir -p bin
71-
GOBIN=`pwd`/bin/ go install gotest.tools/gotestsum@latest
73+
architecture=$(uname -m)
74+
75+
case "$architecture" in
76+
x86_64 | amd64) architecture="amd64" ;;
77+
aarch64 | arm64) architecture="arm64" ;;
78+
*)
79+
echo "Unsupported architecture: $architecture"
80+
exit 1
81+
;;
82+
esac
83+
84+
gotestsum_version="1.13.0"
85+
gotestsum_archive="gotestsum_${gotestsum_version}_linux_${architecture}.tar.gz"
86+
gotestsum_url="https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_version}/${gotestsum_archive}"
87+
88+
temp_file="$(mktemp)"
89+
curl -fsSL "$gotestsum_url" -o "$temp_file"
90+
tar -xzf "$temp_file" -C bin
91+
chmod +x bin/gotestsum
92+
rm -f "$temp_file"
7293

7394
# gotestsum configure to only show logs for failed tests, json file for detailed logs
7495
# Run the tests! Yey!
7596
test_exit_code=0
76-
./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 100 -timeout 90m || test_exit_code=$?
97+
./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 150 -timeout 90m || test_exit_code=$?
7798

7899
# Upload test results as Azure DevOps artifacts
79100
echo "##vso[artifact.upload containerfolder=test-results;artifactname=e2e-test-log]${BUILD_SRC_DIR}/e2e/test-log.json"

.pipelines/templates/e2e-template.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ jobs:
3434
displayName: Run AgentBaker E2E
3535
env:
3636
E2E_SUBSCRIPTION_ID: $(E2E_SUBSCRIPTION_ID)
37+
SYS_SSH_PUBLIC_KEY: $(SYS_SSH_PUBLIC_KEY)
38+
SYS_SSH_PRIVATE_KEY_B64: $(SYS_SSH_PRIVATE_KEY_B64)
3739
BUILD_SRC_DIR: $(System.DefaultWorkingDirectory)
3840
DefaultWorkingDirectory: $(Build.SourcesDirectory)
3941
VHD_BUILD_ID: $(VHD_BUILD_ID)

aks-node-controller/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/Azure/agentbaker/aks-node-controller
22

3-
go 1.23.7
3+
go 1.24.0
44

55
require (
66
github.com/Azure/agentbaker v0.20240503.0
@@ -28,7 +28,7 @@ require (
2828
github.com/pkg/errors v0.9.1 // indirect
2929
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
3030
github.com/vincent-petithory/dataurl v1.0.0 // indirect
31-
golang.org/x/sys v0.35.0 // indirect
31+
golang.org/x/sys v0.39.0 // indirect
3232
gopkg.in/yaml.v3 v3.0.1 // indirect
3333
)
3434

aks-node-controller/go.sum

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8A
5555
github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
5656
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
5757
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
58-
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
59-
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
60-
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
61-
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
62-
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
63-
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
58+
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
59+
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
60+
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
61+
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
62+
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
63+
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
6464
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
6565
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
6666
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

e2e/aks_model.go

Lines changed: 78 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ import (
1313
"github.com/Azure/agentbaker/pkg/agent"
1414
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
1515
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
16-
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
17-
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
18-
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
16+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2"
17+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8"
18+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7"
1919
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
2020
)
2121

@@ -175,6 +175,16 @@ func getBaseClusterModel(clusterName, location, k8sSystemPoolSKU string) *armcon
175175
Enabled: to.Ptr(false),
176176
},
177177
},
178+
LinuxProfile: &armcontainerservice.LinuxProfile{
179+
AdminUsername: to.Ptr("azureuser"),
180+
SSH: &armcontainerservice.SSHConfiguration{
181+
PublicKeys: []*armcontainerservice.SSHPublicKey{
182+
{
183+
KeyData: to.Ptr(string(config.SysSSHPublicKey)),
184+
},
185+
},
186+
},
187+
},
178188
},
179189
Identity: &armcontainerservice.ManagedClusterIdentity{
180190
Type: to.Ptr(armcontainerservice.ResourceIdentityTypeSystemAssigned),
@@ -275,6 +285,17 @@ func addFirewallRules(
275285
ctx context.Context, clusterModel *armcontainerservice.ManagedCluster,
276286
location string,
277287
) error {
288+
routeTableName := "abe2e-fw-rt"
289+
rtGetResp, err := config.Azure.RouteTables.Get(
290+
ctx,
291+
*clusterModel.Properties.NodeResourceGroup,
292+
routeTableName,
293+
nil,
294+
)
295+
if err == nil && len(rtGetResp.Properties.Subnets) != 0 {
296+
// already associated with aks subnet
297+
return nil
298+
}
278299

279300
vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup)
280301
if err != nil {
@@ -366,7 +387,6 @@ func addFirewallRules(
366387
return fmt.Errorf("failed to get firewall private IP address")
367388
}
368389

369-
routeTableName := "abe2e-fw-rt"
370390
routeTableParams := armnetwork.RouteTable{
371391
Location: to.Ptr(location),
372392
Properties: &armnetwork.RouteTablePropertiesFormat{
@@ -535,55 +555,33 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup
535555

536556
func addPrivateEndpointForACR(ctx context.Context, nodeResourceGroup, privateACRName string, vnet VNet, location string) error {
537557
logf(ctx, "Checking if private endpoint for private container registry is in rg %s", nodeResourceGroup)
538-
539558
var err error
540-
var exists bool
559+
var privateEndpoint *armnetwork.PrivateEndpoint
541560
privateEndpointName := "PE-for-ABE2ETests"
542-
if exists, err = privateEndpointExists(ctx, nodeResourceGroup, privateEndpointName); err != nil {
543-
return err
544-
}
545-
if exists {
546-
logf(ctx, "Private Endpoint already exists, skipping creation")
547-
return nil
548-
}
549-
550-
var peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse
551-
if peResp, err = createPrivateEndpoint(ctx, nodeResourceGroup, privateEndpointName, privateACRName, vnet, location); err != nil {
561+
if privateEndpoint, err = createPrivateEndpoint(ctx, nodeResourceGroup, privateEndpointName, privateACRName, vnet, location); err != nil {
552562
return err
553563
}
554564

555565
privateZoneName := "privatelink.azurecr.io"
556-
var pzResp armprivatedns.PrivateZonesClientCreateOrUpdateResponse
557-
if pzResp, err = createPrivateZone(ctx, nodeResourceGroup, privateZoneName); err != nil {
566+
var privateZone *armprivatedns.PrivateZone
567+
if privateZone, err = createPrivateZone(ctx, nodeResourceGroup, privateZoneName); err != nil {
558568
return err
559569
}
560570

561571
if err = createPrivateDNSLink(ctx, vnet, nodeResourceGroup, privateZoneName); err != nil {
562572
return err
563573
}
564574

565-
if err = addRecordSetToPrivateDNSZone(ctx, peResp, nodeResourceGroup, privateZoneName); err != nil {
575+
if err = addRecordSetToPrivateDNSZone(ctx, privateEndpoint, nodeResourceGroup, privateZoneName); err != nil {
566576
return err
567577
}
568578

569-
if err = addDNSZoneGroup(ctx, pzResp, nodeResourceGroup, privateZoneName, *peResp.Name); err != nil {
579+
if err = addDNSZoneGroup(ctx, privateZone, nodeResourceGroup, privateZoneName, *privateEndpoint.Name); err != nil {
570580
return err
571581
}
572582
return nil
573583
}
574584

575-
func privateEndpointExists(ctx context.Context, nodeResourceGroup, privateEndpointName string) (bool, error) {
576-
existingPE, err := config.Azure.PrivateEndpointClient.Get(ctx, nodeResourceGroup, privateEndpointName, nil)
577-
if err == nil && existingPE.ID != nil {
578-
logf(ctx, "Private Endpoint already exists with ID: %s", *existingPE.ID)
579-
return true, nil
580-
}
581-
if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") {
582-
return false, fmt.Errorf("failed to get private endpoint: %w", err)
583-
}
584-
return false, nil
585-
}
586-
587585
func createPrivateAzureContainerRegistryPullSecret(ctx context.Context, cluster *armcontainerservice.ManagedCluster, kubeconfig *Kubeclient, resourceGroup string, isNonAnonymousPull bool) error {
588586
privateACRName := config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location)
589587
if isNonAnonymousPull {
@@ -768,7 +766,15 @@ func addCacheRulesToPrivateAzureContainerRegistry(ctx context.Context, resourceG
768766
return nil
769767
}
770768

771-
func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpointName, privateACRName string, vnet VNet, location string) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
769+
func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpointName, privateACRName string, vnet VNet, location string) (*armnetwork.PrivateEndpoint, error) {
770+
existingPE, err := config.Azure.PrivateEndpointClient.Get(ctx, nodeResourceGroup, privateEndpointName, nil)
771+
if err == nil && existingPE.ID != nil {
772+
logf(ctx, "Private Endpoint already exists with ID: %s", *existingPE.ID)
773+
return &existingPE.PrivateEndpoint, nil
774+
}
775+
if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") {
776+
return nil, fmt.Errorf("failed to get private endpoint: %w", err)
777+
}
772778
logf(ctx, "Creating Private Endpoint in rg %s", nodeResourceGroup)
773779
acrID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName(location), privateACRName)
774780

@@ -798,18 +804,27 @@ func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpoi
798804
nil,
799805
)
800806
if err != nil {
801-
return armnetwork.PrivateEndpointsClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private endpoint in BeginCreateOrUpdate: %w", err)
807+
return nil, fmt.Errorf("failed to create private endpoint in BeginCreateOrUpdate: %w", err)
802808
}
803809
resp, err := poller.PollUntilDone(ctx, nil)
804810
if err != nil {
805-
return armnetwork.PrivateEndpointsClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private endpoint in polling: %w", err)
811+
return nil, fmt.Errorf("failed to create private endpoint in polling: %w", err)
806812
}
807813

808814
logf(ctx, "Private Endpoint created or updated with ID: %s", *resp.ID)
809-
return resp, nil
815+
return &resp.PrivateEndpoint, nil
810816
}
811817

812-
func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName string) (armprivatedns.PrivateZonesClientCreateOrUpdateResponse, error) {
818+
func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName string) (*armprivatedns.PrivateZone, error) {
819+
pzResp, err := config.Azure.PrivateZonesClient.Get(
820+
ctx,
821+
nodeResourceGroup,
822+
privateZoneName,
823+
nil,
824+
)
825+
if err == nil {
826+
return &pzResp.PrivateZone, nil
827+
}
813828
dnsZoneParams := armprivatedns.PrivateZone{
814829
Location: to.Ptr("global"),
815830
}
@@ -821,23 +836,36 @@ func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName s
821836
nil,
822837
)
823838
if err != nil {
824-
return armprivatedns.PrivateZonesClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private dns zone in BeginCreateOrUpdate: %w", err)
839+
return nil, fmt.Errorf("failed to create private dns zone in BeginCreateOrUpdate: %w", err)
825840
}
826841
resp, err := poller.PollUntilDone(ctx, nil)
827842
if err != nil {
828-
return armprivatedns.PrivateZonesClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private dns zone in polling: %w", err)
843+
return nil, fmt.Errorf("failed to create private dns zone in polling: %w", err)
829844
}
830845

831846
logf(ctx, "Private DNS Zone created or updated with ID: %s", *resp.ID)
832-
return resp, nil
847+
return &resp.PrivateZone, nil
833848
}
834849

835850
func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, privateZoneName string) error {
851+
networkLinkName := "link-ABE2ETests"
852+
_, err := config.Azure.VirutalNetworkLinksClient.Get(
853+
ctx,
854+
nodeResourceGroup,
855+
privateZoneName,
856+
networkLinkName,
857+
nil,
858+
)
859+
860+
if err == nil {
861+
// private dns link already created
862+
return nil
863+
}
864+
836865
vnetForId, err := config.Azure.VNet.Get(ctx, nodeResourceGroup, vnet.name, nil)
837866
if err != nil {
838867
return fmt.Errorf("failed to get vnet: %w", err)
839868
}
840-
networkLinkName := "link-ABE2ETests"
841869
linkParams := armprivatedns.VirtualNetworkLink{
842870
Location: to.Ptr("global"),
843871
Properties: &armprivatedns.VirtualNetworkLinkProperties{
@@ -867,16 +895,16 @@ func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, pri
867895
return nil
868896
}
869897

870-
func addRecordSetToPrivateDNSZone(ctx context.Context, peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, nodeResourceGroup, privateZoneName string) error {
871-
for i, dnsConfigPtr := range peResp.Properties.CustomDNSConfigs {
898+
func addRecordSetToPrivateDNSZone(ctx context.Context, privateEndpoint *armnetwork.PrivateEndpoint, nodeResourceGroup, privateZoneName string) error {
899+
for i, dnsConfigPtr := range privateEndpoint.Properties.CustomDNSConfigs {
872900
var ipAddresses []string
873901
if dnsConfigPtr == nil {
874902
return fmt.Errorf("CustomDNSConfigs[%d] is nil", i)
875903
}
876904

877905
// get the ip addresses
878906
dnsConfig := *dnsConfigPtr
879-
if dnsConfig.IPAddresses == nil || len(dnsConfig.IPAddresses) == 0 {
907+
if len(dnsConfig.IPAddresses) == 0 {
880908
return fmt.Errorf("CustomDNSConfigs[%d].IPAddresses is nil or empty", i)
881909
}
882910
for _, ipPtr := range dnsConfig.IPAddresses {
@@ -907,15 +935,19 @@ func addRecordSetToPrivateDNSZone(ctx context.Context, peResp armnetwork.Private
907935
return nil
908936
}
909937

910-
func addDNSZoneGroup(ctx context.Context, pzResp armprivatedns.PrivateZonesClientCreateOrUpdateResponse, nodeResourceGroup, privateZoneName, endpointName string) error {
938+
func addDNSZoneGroup(ctx context.Context, privateZone *armprivatedns.PrivateZone, nodeResourceGroup, privateZoneName, endpointName string) error {
911939
groupName := strings.Replace(privateZoneName, ".", "-", -1) // replace . with -
940+
_, err := config.Azure.PrivateDNSZoneGroup.Get(ctx, nodeResourceGroup, endpointName, groupName, nil)
941+
if err == nil {
942+
return nil
943+
}
912944
dnsZonegroup := armnetwork.PrivateDNSZoneGroup{
913945
Name: to.Ptr(fmt.Sprintf("%s/default", privateZoneName)),
914946
Properties: &armnetwork.PrivateDNSZoneGroupPropertiesFormat{
915947
PrivateDNSZoneConfigs: []*armnetwork.PrivateDNSZoneConfig{{
916948
Name: to.Ptr(groupName),
917949
Properties: &armnetwork.PrivateDNSZonePropertiesFormat{
918-
PrivateDNSZoneID: pzResp.ID,
950+
PrivateDNSZoneID: privateZone.ID,
919951
},
920952
}},
921953
},

0 commit comments

Comments
 (0)