Skip to content

Commit

Permalink
MGMT-19120: Use service net to connect to hosted API server
Browse files Browse the repository at this point in the history 
There are several situations where assisted service needs to connect to
the API server of a spoke cluster. To do so it uses the kubeconfig
generated during the installation, and that usually contains the
external URL of the API server, and that means that the cluster where
assisted service runs needs to be configured with a proxy that allows
that. But for HyperShift clusters this can be avoided: assisted service
can instead connect via the service network, using the
`kube-apiserver.my-cluster.svc` host name, as the API server runs as a
pod in the same cluster. Duing that reduces the number of round trips
and the potential proxy configuration issues. In order to achive that
this patch changes the spoke client factory so that it checks if the
cluster is a HyperShift cluster, and then it replaces the API server URL
with `https://kube-apiserver.my-cluster.svc:6443`.

Related: https://issues.redhat.com/browse/MGMT-19120
Signed-off-by: Juan Hernandez <[email protected]>
  • Loading branch information
@jhernand
jhernand committed Dec 13, 2024
1 parent 0827142 commit 63c3673
Show file tree
Hide file tree
Showing 42 changed files with 4,529 additions and 357 deletions.
12 changes: 9 additions & 3 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,12 @@ func main() {
InsecureIPXEURLs: generateInsecureIPXEURLs,
}).SetupWithManager(ctrlMgr), "unable to create controller InfraEnv")

spokeClientFactory, err := spoke_k8s_client.NewFactory().
SetLogger(log).
SetHubClient(c).
Build()
failOnError(err, "unable to create spoke client factory")

cluster_client := ctrlMgr.GetClient()
cluster_reader := ctrlMgr.GetAPIReader()
failOnError((&controllers.ClusterDeploymentsReconciler{
Expand All @@ -636,7 +642,7 @@ func main() {
PullSecretHandler: controllers.NewPullSecretHandler(cluster_client, cluster_reader, bm),
AuthType: Options.Auth.AuthType,
VersionsHandler: versionHandler,
SpokeK8sClientFactory: spoke_k8s_client.NewSpokeK8sClientFactory(log),
SpokeK8sClientFactory: spokeClientFactory,
MirrorRegistriesConfigBuilder: mirrorregistries.New(),
}).SetupWithManager(ctrlMgr), "unable to create controller ClusterDeployment")

Expand All @@ -649,7 +655,7 @@ func main() {
CRDEventsHandler: crdEventsHandler,
ServiceBaseURL: Options.BMConfig.ServiceBaseURL,
AuthType: Options.Auth.AuthType,
SpokeK8sClientFactory: spoke_k8s_client.NewSpokeK8sClientFactory(log),
SpokeK8sClientFactory: spokeClientFactory,
ApproveCsrsRequeueDuration: Options.ApproveCsrsRequeueDuration,
AgentContainerImage: Options.BMConfig.AgentDockerImg,
HostFSMountDir: hostFSMountDir,
Expand All @@ -661,7 +667,7 @@ func main() {
Log: log,
Scheme: ctrlMgr.GetScheme(),
Installer: bm,
SpokeK8sClientFactory: spoke_k8s_client.NewSpokeK8sClientFactory(log),
SpokeK8sClientFactory: spokeClientFactory,
ConvergedFlowEnabled: useConvergedFlow,
PauseProvisionedBMHs: Options.PauseProvisionedBMHs,
Drainer: &controllers.KubectlDrainer{},
Expand Down
9 changes: 8 additions & 1 deletion cmd/operator/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,14 @@ func main() {
}

log := logrus.New()
spokeClientFactory := spoke_k8s_client.NewSpokeK8sClientFactory(log)
spokeClientFactory, err := spoke_k8s_client.NewFactory().
SetLogger(log).
SetHubClient(mgr.GetClient()).
Build()
if err != nil {
log.WithError(err).Error("failed to create spoke client factory")
os.Exit(1)
}
spokeClientCache := controllers.NewSpokeClientCache(spokeClientFactory)

c, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
Expand Down
19 changes: 4 additions & 15 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
Expand Down Expand Up @@ -622,9 +623,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU=
github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
Expand Down Expand Up @@ -966,9 +966,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand All @@ -978,8 +977,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
Expand Down Expand Up @@ -1419,8 +1416,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
Expand All @@ -1447,8 +1442,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
Expand Down Expand Up @@ -1671,8 +1664,6 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -1773,8 +1764,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
Expand Down
1 change: 1 addition & 0 deletions hack/setup_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ function test_tools() {
go install gotest.tools/[email protected]
go install github.com/axw/gocov/[email protected]
go install github.com/AlekSi/[email protected]
go install sigs.k8s.io/controller-runtime/tools/[email protected]
}

function assisted_service() {
Expand Down
20 changes: 12 additions & 8 deletions internal/controller/controllers/agent_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ type AgentReconciler struct {
CRDEventsHandler CRDEventsHandler
ServiceBaseURL string
AuthType auth.AuthType
SpokeK8sClientFactory spoke_k8s_client.SpokeK8sClientFactory
SpokeK8sClientFactory spoke_k8s_client.Factory
ApproveCsrsRequeueDuration time.Duration
AgentContainerImage string
HostFSMountDir string
Expand Down Expand Up @@ -360,7 +360,7 @@ func deleteBMHForMachine(ctx context.Context, spokeClient client.Client, machine

// removeSpokeResources removes all relevant resources from the agent's spoke cluster
// This includes all or some of the node, machine, BMH, and scaling the machineset depending on what is present
func removeSpokeResources(ctx context.Context, log logrus.FieldLogger, spokeClient spoke_k8s_client.SpokeK8sClient, nodeName string) error {
func removeSpokeResources(ctx context.Context, log logrus.FieldLogger, spokeClient spoke_k8s_client.Client, nodeName string) error {
log = log.WithField("node", nodeName)

nodeKey := client.ObjectKey{Name: nodeName}
Expand Down Expand Up @@ -540,7 +540,7 @@ func (r *AgentReconciler) shouldApproveCSR(csr *certificatesv1.CertificateSignin
return validateNodeCsr(agent, csr, x509CSR)
}

func (r *AgentReconciler) approveAIHostsCSRs(ctx context.Context, clients spoke_k8s_client.SpokeK8sClient, agent *aiv1beta1.Agent, validateNodeCsr nodeCsrValidator) {
func (r *AgentReconciler) approveAIHostsCSRs(ctx context.Context, clients spoke_k8s_client.Client, agent *aiv1beta1.Agent, validateNodeCsr nodeCsrValidator) {
csrList, err := clients.ListCsrs(ctx)
if err != nil {
r.Log.WithError(err).Errorf("Failed to get CSRs for agent %s/%s", agent.Namespace, agent.Name)
Expand All @@ -565,18 +565,22 @@ func (r *AgentReconciler) approveAIHostsCSRs(ctx context.Context, clients spoke_
}
}

func (r *AgentReconciler) spokeKubeClient(ctx context.Context, clusterRef *aiv1beta1.ClusterReference) (spoke_k8s_client.SpokeK8sClient, error) {
func (r *AgentReconciler) spokeKubeClient(ctx context.Context, clusterRef *aiv1beta1.ClusterReference) (spoke_k8s_client.Client, error) {
secret, err := spokeKubeconfigSecret(ctx, r.Log, r.Client, r.APIReader, clusterRef)
if err != nil {
r.Log.WithError(err).Errorf("failed to get spoke secret for cluster %s/%s", clusterRef.Namespace, clusterRef.Name)
return nil, err
}
return r.SpokeK8sClientFactory.CreateFromSecret(secret)
clusterKey := types.NamespacedName{
Namespace: clusterRef.Namespace,
Name: clusterRef.Name,
}
return r.SpokeK8sClientFactory.CreateFromSecret(ctx, clusterKey, secret)
}

// Attempt to approve CSRs for agent. If already approved then the node will be marked as done
// requeue means that approval will be attempted again
func (r *AgentReconciler) tryApproveDay2CSRs(ctx context.Context, agent *aiv1beta1.Agent, node *corev1.Node, client spoke_k8s_client.SpokeK8sClient) {
func (r *AgentReconciler) tryApproveDay2CSRs(ctx context.Context, agent *aiv1beta1.Agent, node *corev1.Node, client spoke_k8s_client.Client) {
r.Log.Infof("Approving CSRs for agent %s/%s", agent.Namespace, agent.Name)
var validateNodeCsr nodeCsrValidator

Expand Down Expand Up @@ -771,7 +775,7 @@ func marshalNodeLabels(nodeLabels map[string]string) (string, error) {
return string(b), err
}

func (r *AgentReconciler) applyDay2NodeLabels(ctx context.Context, log logrus.FieldLogger, agent *aiv1beta1.Agent, node *corev1.Node, client spoke_k8s_client.SpokeK8sClient) error {
func (r *AgentReconciler) applyDay2NodeLabels(ctx context.Context, log logrus.FieldLogger, agent *aiv1beta1.Agent, node *corev1.Node, client spoke_k8s_client.Client) error {
if funk.IsEmpty(agent.Spec.NodeLabels) || !isNodeReady(node) {
return nil
}
Expand Down Expand Up @@ -805,7 +809,7 @@ func (r *AgentReconciler) updateStatus(ctx context.Context, log logrus.FieldLogg
var (
err error
shouldAutoApproveCSRs bool
spokeClient spoke_k8s_client.SpokeK8sClient
spokeClient spoke_k8s_client.Client
node *corev1.Node
)
ret := ctrl.Result{}
Expand Down
Loading

0 comments on commit 63c3673

Please sign in to comment.