Skip to content

Commit 52a7727

Browse files
iQQBotkylos101
iQQBot
and
kylos101
authored
[node-labeler] Refactor node labeling to use taints instead of labels (#20652)
* [node-labeler] Refactor node labeling to use taints instead of labels * [agent-smith] Add toleration to daemonset * Add workspace component tolerations to various Gitpod components if it running in Full installation * Apply suggestions from code review Co-authored-by: Kyle Brennan <kyle@gitpod.io> * Update components/node-labeler/cmd/run.go Co-authored-by: Kyle Brennan <kyle@gitpod.io> --------- Co-authored-by: Kyle Brennan <kyle@gitpod.io>
1 parent 1becba7 commit 52a7727

File tree

26 files changed

+383
-71
lines changed

26 files changed

+383
-71
lines changed

‎components/node-labeler/cmd/run.go

+276-68
Large diffs are not rendered by default.

‎install/installer/pkg/common/toleration.go

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright (c) 2025 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
package common
6+
7+
import (
8+
configv1 "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
9+
corev1 "k8s.io/api/core/v1"
10+
)
11+
12+
const (
13+
RegistryFacadeTaintKey = "gitpod.io/registry-facade-not-ready"
14+
WsDaemonTaintKey = "gitpod.io/ws-daemon-not-ready"
15+
)
16+
17+
func WithTolerationWorkspaceComponentNotReady(ctx *RenderContext) []corev1.Toleration {
18+
if ctx.Config.Kind != configv1.InstallationFull {
19+
return []corev1.Toleration{}
20+
}
21+
return []corev1.Toleration{
22+
{
23+
Key: RegistryFacadeTaintKey,
24+
Operator: corev1.TolerationOpExists,
25+
Effect: corev1.TaintEffectNoSchedule,
26+
},
27+
{
28+
Key: WsDaemonTaintKey,
29+
Operator: corev1.TolerationOpExists,
30+
Effect: corev1.TaintEffectNoSchedule,
31+
},
32+
}
33+
}

‎install/installer/pkg/components/agent-smith/daemonset.go

+6
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,12 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
105105
},
106106
common.CAVolume(),
107107
},
108+
Tolerations: []corev1.Toleration{
109+
{
110+
Effect: corev1.TaintEffectNoSchedule,
111+
Operator: corev1.TolerationOpExists,
112+
},
113+
},
108114
},
109115
},
110116
UpdateStrategy: common.DaemonSetRolloutStrategy(),

‎install/installer/pkg/components/blobserve/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
168168
FailureThreshold: 3,
169169
},
170170
}, *common.KubeRBACProxyContainer(ctx)},
171+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
171172
},
172173
},
173174
},

‎install/installer/pkg/components/content-service/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
8888
},
8989
}, *common.KubeRBACProxyContainer(ctx),
9090
},
91+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
9192
}
9293

9394
err = common.AddStorageMounts(ctx, &podSpec, Component)

‎install/installer/pkg/components/dashboard/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
8686
TimeoutSeconds: 1,
8787
},
8888
}},
89+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
8990
},
9091
},
9192
},

‎install/installer/pkg/components/database/incluster/helm.go

+10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,15 @@ var Helm = common.CompositeHelmFunc(
2525
return nil, err
2626
}
2727

28+
tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
29+
if err != nil {
30+
return nil, err
31+
}
32+
tolerationsTemplate, err := helm.KeyFileValue("mysql.primary.tolerations", tolerations)
33+
if err != nil {
34+
return nil, err
35+
}
36+
2837
imageRegistry := common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)
2938

3039
type EnvVar struct {
@@ -73,6 +82,7 @@ var Helm = common.CompositeHelmFunc(
7382
FileValues: []string{
7483
primaryAffinityTemplate,
7584
extraEnvVarsTemplate,
85+
tolerationsTemplate,
7686
},
7787
},
7888
}, nil

‎install/installer/pkg/components/docker-registry/helm.go

+13
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,25 @@ var Helm = common.CompositeHelmFunc(
6565
)
6666
}
6767

68+
tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
69+
if err != nil {
70+
return nil, err
71+
}
72+
tolerationsTemplate, err := helm.KeyFileValue("docker-registry.tolerations", tolerations)
73+
if err != nil {
74+
return nil, err
75+
}
76+
6877
registryValues = append(registryValues, helm.KeyValue("docker-registry.persistence.enabled", enablePersistence))
6978

7079
return &common.HelmConfig{
7180
Enabled: inCluster,
7281
Values: &values.Options{
7382
Values: registryValues,
83+
// This is too complex to be sent as a string
84+
FileValues: []string{
85+
tolerationsTemplate,
86+
},
7487
},
7588
}, nil
7689
}),

‎install/installer/pkg/components/ide-metrics/deployment.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
120120
},
121121
*common.KubeRBACProxyContainerWithConfig(ctx),
122122
},
123-
Volumes: volumes,
123+
Volumes: volumes,
124+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
124125
},
125126
},
126127
},

‎install/installer/pkg/components/ide-proxy/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
8282
TimeoutSeconds: 1,
8383
},
8484
}},
85+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
8586
},
8687
},
8788
},

‎install/installer/pkg/components/ide-service/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
153153
},
154154
},
155155
},
156+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
156157
},
157158
},
158159
},

‎install/installer/pkg/components/image-builder-mk3/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
188188
},
189189
*common.KubeRBACProxyContainer(ctx),
190190
},
191+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
191192
},
192193
},
193194
},

‎install/installer/pkg/components/migrations/job.go

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ func job(ctx *common.RenderContext) ([]runtime.Object, error) {
5656
"cd /app/node_modules/@gitpod/gitpod-db && yarn run wait-for-db && yarn run typeorm migration:show || true && yarn run typeorm migration:run",
5757
},
5858
}},
59+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
5960
},
6061
},
6162
},

‎install/installer/pkg/components/minio/incluster/minio.go

+11
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package incluster
66

77
import (
88
"fmt"
9+
910
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
1011
"github.com/gitpod-io/gitpod/installer/pkg/common"
1112
"github.com/gitpod-io/gitpod/installer/pkg/helm"
@@ -26,6 +27,15 @@ var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) com
2627
return nil, err
2728
}
2829

30+
tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
31+
if err != nil {
32+
return nil, err
33+
}
34+
tolerationsTemplate, err := helm.KeyFileValue("minio.tolerations", tolerations)
35+
if err != nil {
36+
return nil, err
37+
}
38+
2939
return &common.HelmConfig{
3040
Enabled: true,
3141
Values: &values.Options{
@@ -41,6 +51,7 @@ var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) com
4151
// This is too complex to be sent as a string
4252
FileValues: []string{
4353
affinityTemplate,
54+
tolerationsTemplate,
4455
},
4556
},
4657
}, nil

‎install/installer/pkg/components/node-labeler/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
7171
},
7272
*common.KubeRBACProxyContainerWithConfig(ctx),
7373
},
74+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
7475
}
7576

7677
return []runtime.Object{

‎install/installer/pkg/components/openvsx-proxy/statefulset.go

+1
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
195195
}},
196196
}, *common.KubeRBACProxyContainer(ctx),
197197
},
198+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
198199
},
199200
},
200201
VolumeClaimTemplates: volumeClaimTemplates,

‎install/installer/pkg/components/proxy/deployment.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,9 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
286286
Value: segmentEndpoint,
287287
}},
288288
)),
289-
}},
289+
},
290+
},
291+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
290292
},
291293
},
292294
},

‎install/installer/pkg/components/public-api-server/deployment.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,8 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
187187
},
188188
*common.KubeRBACProxyContainer(ctx),
189189
},
190-
Volumes: volumes,
190+
Volumes: volumes,
191+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
191192
},
192193
},
193194
},

‎install/installer/pkg/components/redis/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
143143
},
144144
*common.KubeRBACProxyContainer(ctx),
145145
},
146+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
146147
},
147148
},
148149
},

‎install/installer/pkg/components/server/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
416416
volumeMounts...,
417417
),
418418
}, *common.KubeRBACProxyContainer(ctx)},
419+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
419420
},
420421
},
421422
},

‎install/installer/pkg/components/spicedb/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
189189
},
190190
*common.KubeRBACProxyContainer(ctx),
191191
},
192+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
192193
Volumes: []v1.Volume{
193194
bootstrapVolume,
194195
},

‎install/installer/pkg/components/usage/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
152152
},
153153
*common.KubeRBACProxyContainerWithConfig(ctx),
154154
},
155+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
155156
},
156157
},
157158
},

‎install/installer/pkg/components/ws-manager-bridge/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
209209
PeriodSeconds: 20,
210210
},
211211
}, *common.KubeRBACProxyContainer(ctx)},
212+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
212213
},
213214
},
214215
},

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
146146
},
147147
*common.KubeRBACProxyContainer(ctx),
148148
},
149+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
149150
Volumes: append([]corev1.Volume{
150151
{
151152
Name: VolumeConfig,

‎install/installer/pkg/components/ws-proxy/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
184184
},
185185
*common.KubeRBACProxyContainer(ctx),
186186
},
187+
Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
187188
}
188189

189190
return []runtime.Object{

‎install/installer/pkg/helm/helm.go

+11
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,17 @@ func AffinityYaml(orLabels ...string) ([]byte, error) {
125125
return marshal, nil
126126
}
127127

128+
func WithTolerationWorkspaceComponentNotReadyYaml(ctx *common.RenderContext) ([]byte, error) {
129+
tolerations := common.WithTolerationWorkspaceComponentNotReady(ctx)
130+
131+
marshal, err := yaml.Marshal(tolerations)
132+
if err != nil {
133+
return nil, err
134+
}
135+
136+
return marshal, nil
137+
}
138+
128139
func nodeAffinity(orLabels ...string) *corev1.Affinity {
129140
var terms []corev1.NodeSelectorTerm
130141
for _, lbl := range orLabels {

0 commit comments

Comments
 (0)