From f29f66b49f5d1f099d6d0bb58509692f854ffd91 Mon Sep 17 00:00:00 2001 From: Stephen Kitt Date: Thu, 17 Dec 2020 12:28:56 +0100 Subject: [PATCH] Handle the broker in the operator Fixes: #203 Signed-off-by: Stephen Kitt --- .yamllint.yml | 2 + apis/submariner/v1alpha1/submariner_types.go | 46 +++++ .../v1alpha1/zz_generated.deepcopy.go | 99 +++++++++ config/crd/bases/submariner.io_brokers.yaml | 67 ++++++ config/crd/kustomization.yaml | 3 + .../crd/patches/cainjection_in_brokers.yaml | 8 + config/crd/patches/webhook_in_brokers.yaml | 17 ++ config/samples/kustomization.yaml | 1 + .../samples/submariner_v1alpha1_broker.yaml | 10 + controllers/submariner/broker_controller.go | 94 +++++++++ main.go | 24 ++- pkg/broker/ensure.go | 26 +-- .../typed/submariner/v1alpha1/broker.go | 191 ++++++++++++++++++ .../submariner/v1alpha1/fake/fake_broker.go | 140 +++++++++++++ .../v1alpha1/fake/fake_submariner_client.go | 4 + .../v1alpha1/generated_expansion.go | 2 + .../submariner/v1alpha1/submariner_client.go | 5 + pkg/subctl/cmd/deploybroker.go | 45 ++++- pkg/subctl/operator/brokercr/ensure.go | 74 +++++++ .../embeddedyamls/generators/yamls2go.go | 1 + .../operator/submarinerop/crds/ensure.go | 9 +- 21 files changed, 841 insertions(+), 27 deletions(-) create mode 100644 config/crd/bases/submariner.io_brokers.yaml create mode 100644 config/crd/patches/cainjection_in_brokers.yaml create mode 100644 config/crd/patches/webhook_in_brokers.yaml create mode 100644 config/samples/submariner_v1alpha1_broker.yaml create mode 100644 controllers/submariner/broker_controller.go create mode 100644 pkg/client/clientset/versioned/typed/submariner/v1alpha1/broker.go create mode 100644 pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_broker.go create mode 100644 pkg/subctl/operator/brokercr/ensure.go diff --git a/.yamllint.yml b/.yamllint.yml index b3a759048..b6821603c 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -13,10 +13,12 @@ ignore: | /deploy/submariner/crds/submariner.io_gateways.yaml /deploy/submariner/crds/submariner.io_endpoints.yaml /deploy/submariner/crds/submariner.io_clusters.yaml + /deploy/crds/submariner.io_brokers.yaml /deploy/crds/submariner.io_submariners.yaml /deploy/crds/submariner.io_servicediscoveries.yaml /deploy/mcsapi/crds/multicluster.x_k8s.io_serviceexports.yaml /deploy/mcsapi/crds/multicluster.x_k8s.io_serviceimports.yaml + /config/crd/bases/submariner.io_brokers.yaml /config/crd/bases/submariner.io_submariners.yaml /config/crd/bases/submariner.io_servicediscoveries.yaml /config/manager/kustomization.yaml diff --git a/apis/submariner/v1alpha1/submariner_types.go b/apis/submariner/v1alpha1/submariner_types.go index 2188d0f72..efd7d48a5 100644 --- a/apis/submariner/v1alpha1/submariner_types.go +++ b/apis/submariner/v1alpha1/submariner_types.go @@ -126,8 +126,54 @@ type SubmarinerList struct { Items []Submariner `json:"items"` } +// BrokerSpec defines the desired state of Broker +// +k8s:openapi-gen=true +type BrokerSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + Components []string `json:"components,omitempty"` + DefaultCustomDomains []string `json:"defaultCustomDomains,omitempty"` + GlobalnetCIDRRange string `json:"globalnetCIDRRange,omitempty"` + DefaultGlobalnetClusterSize uint `json:"defaultGlobalnetClusterSize,omitempty"` + GlobalnetEnabled bool `json:"globalnetEnabled,omitempty"` +} + +// BrokerStatus defines the observed state of Broker +// +k8s:openapi-gen=true +type BrokerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true + +// Broker is the Schema for the brokers API +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=brokers,scope=Namespaced +// +genclient +// +operator-sdk:csv:customresourcedefinitions:displayName="Broker" +type Broker struct { //nolint:maligned // we want to keep the traditional order + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BrokerSpec `json:"spec,omitempty"` + Status BrokerStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// BrokerList contains a list of Broker +type BrokerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Broker `json:"items"` +} + func init() { SchemeBuilder.Register(&Submariner{}, &SubmarinerList{}) + SchemeBuilder.Register(&Broker{}, &BrokerList{}) } func (s *Submariner) UnmarshalJSON(data []byte) error { diff --git a/apis/submariner/v1alpha1/zz_generated.deepcopy.go b/apis/submariner/v1alpha1/zz_generated.deepcopy.go index 2ea42d9f2..7c784d284 100644 --- a/apis/submariner/v1alpha1/zz_generated.deepcopy.go +++ b/apis/submariner/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,105 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Broker) DeepCopyInto(out *Broker) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Broker. +func (in *Broker) DeepCopy() *Broker { + if in == nil { + return nil + } + out := new(Broker) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Broker) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerList) DeepCopyInto(out *BrokerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Broker, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerList. +func (in *BrokerList) DeepCopy() *BrokerList { + if in == nil { + return nil + } + out := new(BrokerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BrokerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerSpec) DeepCopyInto(out *BrokerSpec) { + *out = *in + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DefaultCustomDomains != nil { + in, out := &in.DefaultCustomDomains, &out.DefaultCustomDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerSpec. +func (in *BrokerSpec) DeepCopy() *BrokerSpec { + if in == nil { + return nil + } + out := new(BrokerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerStatus) DeepCopyInto(out *BrokerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerStatus. +func (in *BrokerStatus) DeepCopy() *BrokerStatus { + if in == nil { + return nil + } + out := new(BrokerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) { *out = *in diff --git a/config/crd/bases/submariner.io_brokers.yaml b/config/crd/bases/submariner.io_brokers.yaml new file mode 100644 index 000000000..c8bd0f5d5 --- /dev/null +++ b/config/crd/bases/submariner.io_brokers.yaml @@ -0,0 +1,67 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: brokers.submariner.io +spec: + group: submariner.io + names: + kind: Broker + listKind: BrokerList + plural: brokers + singular: broker + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Broker is the Schema for the brokers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BrokerSpec defines the desired state of Broker + properties: + components: + items: + type: string + type: array + defaultCustomDomains: + items: + type: string + type: array + defaultGlobalnetClusterSize: + type: integer + globalnetCIDRRange: + type: string + globalnetEnabled: + type: boolean + type: object + status: + description: BrokerStatus defines the observed state of Broker + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d8a59751d..9d6b9040f 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: # - bases/submariner.io_servicediscoveries.yaml - bases/submariner.io_submariners.yaml + - bases/submariner.io_brokers.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -12,12 +13,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_submariners.yaml #- patches/webhook_in_servicediscoveries.yaml +#- patches/webhook_in_brokers.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_submariners.yaml #- patches/cainjection_in_servicediscoveries.yaml +#- patches/cainjection_in_brokers.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_brokers.yaml b/config/crd/patches/cainjection_in_brokers.yaml new file mode 100644 index 000000000..46cc69696 --- /dev/null +++ b/config/crd/patches/cainjection_in_brokers.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: brokers.submariner.io diff --git a/config/crd/patches/webhook_in_brokers.yaml b/config/crd/patches/webhook_in_brokers.yaml new file mode 100644 index 000000000..860fd39a4 --- /dev/null +++ b/config/crd/patches/webhook_in_brokers.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: brokers.submariner.io +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index d1f7d9d08..6a1895157 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -3,6 +3,7 @@ resources: - lighthouse_v2alpha1_serviceexport.yaml - lighthouse_v2alpha1_serviceimport.yaml + - submariner_v1alpha1_broker.yaml - submariner_v1alpha1_submariner.yaml - submariner_v1alpha1_servicediscovery.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/submariner_v1alpha1_broker.yaml b/config/samples/submariner_v1alpha1_broker.yaml new file mode 100644 index 000000000..aac5ff794 --- /dev/null +++ b/config/samples/submariner_v1alpha1_broker.yaml @@ -0,0 +1,10 @@ +apiVersion: submariner.io/v1alpha1 +kind: Broker +metadata: + name: broker-sample +spec: +# defaultCustomDomains: +# globalnetCIDRRange: + defaultGlobalnetClusterSize: 8192 + globalnetEnabled: false + serviceDiscovery: true diff --git a/controllers/submariner/broker_controller.go b/controllers/submariner/broker_controller.go new file mode 100644 index 000000000..6ca0b387a --- /dev/null +++ b/controllers/submariner/broker_controller.go @@ -0,0 +1,94 @@ +/* +© 2021 Red Hat, Inc. and others. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package submariner + +import ( + "context" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" + "github.com/submariner-io/submariner-operator/pkg/broker" + "github.com/submariner-io/submariner-operator/pkg/engine" + "github.com/submariner-io/submariner-operator/pkg/lighthouse" + crdutils "github.com/submariner-io/submariner-operator/pkg/utils/crds" +) + +// BrokerReconciler reconciles a Broker object +type BrokerReconciler struct { + Client client.Client + Config *rest.Config + Log logr.Logger + Scheme *runtime.Scheme +} + +// TODO skitt: these rbac declarations (and others, see submariner_controller.go) need to be separated +// from methods in order to be taken into account; but they produce ClusterRoles, not the Roles we want +// +kubebuilder:rbac:groups=submariner.io,resources=brokers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=submariner.io,resources=brokers/status,verbs=get;update;patch +func (r *BrokerReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { + _ = context.Background() + _ = r.Log.WithValues("broker", request.NamespacedName) + + // Fetch the Broker instance + instance := &v1alpha1.Broker{} + err := r.Client.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // Broker CRDs + crdUpdater := crdutils.NewFromControllerClient(r.Client) + err = engine.Ensure(crdUpdater) + if err != nil { + return ctrl.Result{}, err + } + + // Lighthouse CRDs + _, err = lighthouse.Ensure(crdUpdater, lighthouse.BrokerCluster) + if err != nil { + return ctrl.Result{}, err + } + + // Globalnet + err = broker.CreateGlobalnetConfigMap(r.Config, instance.Spec.GlobalnetEnabled, instance.Spec.GlobalnetCIDRRange, + instance.Spec.DefaultGlobalnetClusterSize, broker.SubmarinerBrokerNamespace) + if err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.Broker{}). + Complete(r) +} diff --git a/main.go b/main.go index 1f7a2aa31..e9c55db4b 100644 --- a/main.go +++ b/main.go @@ -29,17 +29,19 @@ import ( "k8s.io/client-go/rest" "github.com/operator-framework/operator-lib/leader" + apiruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "github.com/submariner-io/submariner-operator/apis" "github.com/submariner-io/submariner-operator/controllers" + "github.com/submariner-io/submariner-operator/controllers/submariner" "github.com/submariner-io/submariner-operator/pkg/engine" "github.com/submariner-io/submariner-operator/pkg/lighthouse" crdutils "github.com/submariner-io/submariner-operator/pkg/utils/crds" "github.com/submariner-io/submariner-operator/pkg/version" - apiruntime "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" // TODO: in opeartor-sdk v1 the below utilities were moved to internal // TODO: update to operator-sdk v1 or find an alternate way and then change the code accordingly @@ -48,13 +50,14 @@ import ( "github.com/operator-framework/operator-sdk/pkg/metrics" sdkVersion "github.com/operator-framework/operator-sdk/version" "github.com/spf13/pflag" - submarinerv1alpha1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + submarinerv1alpha1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -206,6 +209,15 @@ func main() { log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error()) } } + if err = (&submariner.BrokerReconciler{ + Client: mgr.GetClient(), + Config: mgr.GetConfig(), + Log: ctrl.Log.WithName("controllers").WithName("Broker"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "Broker") + os.Exit(1) + } // +kubebuilder:scaffold:builder // Start the Cmd diff --git a/pkg/broker/ensure.go b/pkg/broker/ensure.go index 5f485c92b..462bbe711 100644 --- a/pkg/broker/ensure.go +++ b/pkg/broker/ensure.go @@ -34,19 +34,21 @@ import ( "k8s.io/client-go/rest" ) -func Ensure(config *rest.Config) error { - crdCreator, err := crdutils.NewFromRestConfig(config) - if err != nil { - return fmt.Errorf("error accessing the target cluster: %s", err) - } - err = engine.Ensure(crdCreator) - if err != nil { - return fmt.Errorf("error setting up the engine requirements: %s", err) - } +func Ensure(config *rest.Config, crds bool) error { + if crds { + crdCreator, err := crdutils.NewFromRestConfig(config) + if err != nil { + return fmt.Errorf("error accessing the target cluster: %s", err) + } + err = engine.Ensure(crdCreator) + if err != nil { + return fmt.Errorf("error setting up the engine requirements: %s", err) + } - _, err = lighthouse.Ensure(crdCreator, lighthouse.BrokerCluster) - if err != nil { - return fmt.Errorf("error setting up the lighthouse requirements: %s", err) + _, err = lighthouse.Ensure(crdCreator, lighthouse.BrokerCluster) + if err != nil { + return fmt.Errorf("error setting up the lighthouse requirements: %s", err) + } } clientset, err := kubernetes.NewForConfig(config) diff --git a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/broker.go b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/broker.go new file mode 100644 index 000000000..20b20d536 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/broker.go @@ -0,0 +1,191 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "time" + + v1alpha1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" + scheme "github.com/submariner-io/submariner-operator/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// BrokersGetter has a method to return a BrokerInterface. +// A group's client should implement this interface. +type BrokersGetter interface { + Brokers(namespace string) BrokerInterface +} + +// BrokerInterface has methods to work with Broker resources. +type BrokerInterface interface { + Create(*v1alpha1.Broker) (*v1alpha1.Broker, error) + Update(*v1alpha1.Broker) (*v1alpha1.Broker, error) + UpdateStatus(*v1alpha1.Broker) (*v1alpha1.Broker, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1alpha1.Broker, error) + List(opts v1.ListOptions) (*v1alpha1.BrokerList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) + BrokerExpansion +} + +// brokers implements BrokerInterface +type brokers struct { + client rest.Interface + ns string +} + +// newBrokers returns a Brokers +func newBrokers(c *SubmarinerV1alpha1Client, namespace string) *brokers { + return &brokers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the broker, and returns the corresponding broker object, and an error if there is any. +func (c *brokers) Get(name string, options v1.GetOptions) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Brokers that match those selectors. +func (c *brokers) List(opts v1.ListOptions) (result *v1alpha1.BrokerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.BrokerList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested brokers. +func (c *brokers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch() +} + +// Create takes the representation of a broker and creates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *brokers) Create(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Post(). + Namespace(c.ns). + Resource("brokers"). + Body(broker). + Do(). + Into(result) + return +} + +// Update takes the representation of a broker and updates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *brokers) Update(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Put(). + Namespace(c.ns). + Resource("brokers"). + Name(broker.Name). + Body(broker). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *brokers) UpdateStatus(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Put(). + Namespace(c.ns). + Resource("brokers"). + Name(broker.Name). + SubResource("status"). + Body(broker). + Do(). + Into(result) + return +} + +// Delete takes name of the broker and deletes it. Returns an error if one occurs. +func (c *brokers) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("brokers"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *brokers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched broker. +func (c *brokers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("brokers"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_broker.go b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_broker.go new file mode 100644 index 000000000..be6fe1c76 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_broker.go @@ -0,0 +1,140 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeBrokers implements BrokerInterface +type FakeBrokers struct { + Fake *FakeSubmarinerV1alpha1 + ns string +} + +var brokersResource = schema.GroupVersionResource{Group: "submariner.io", Version: "v1alpha1", Resource: "brokers"} + +var brokersKind = schema.GroupVersionKind{Group: "submariner.io", Version: "v1alpha1", Kind: "Broker"} + +// Get takes name of the broker, and returns the corresponding broker object, and an error if there is any. +func (c *FakeBrokers) Get(name string, options v1.GetOptions) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(brokersResource, c.ns, name), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// List takes label and field selectors, and returns the list of Brokers that match those selectors. +func (c *FakeBrokers) List(opts v1.ListOptions) (result *v1alpha1.BrokerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(brokersResource, brokersKind, c.ns, opts), &v1alpha1.BrokerList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.BrokerList{ListMeta: obj.(*v1alpha1.BrokerList).ListMeta} + for _, item := range obj.(*v1alpha1.BrokerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested brokers. +func (c *FakeBrokers) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(brokersResource, c.ns, opts)) + +} + +// Create takes the representation of a broker and creates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *FakeBrokers) Create(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(brokersResource, c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// Update takes the representation of a broker and updates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *FakeBrokers) Update(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(brokersResource, c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeBrokers) UpdateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(brokersResource, "status", c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// Delete takes name of the broker and deletes it. Returns an error if one occurs. +func (c *FakeBrokers) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(brokersResource, c.ns, name), &v1alpha1.Broker{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeBrokers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(brokersResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1alpha1.BrokerList{}) + return err +} + +// Patch applies the patch and returns the patched broker. +func (c *FakeBrokers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(brokersResource, c.ns, name, pt, data, subresources...), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} diff --git a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_submariner_client.go b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_submariner_client.go index 462f83422..fe1b0146f 100644 --- a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_submariner_client.go +++ b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/fake/fake_submariner_client.go @@ -28,6 +28,10 @@ type FakeSubmarinerV1alpha1 struct { *testing.Fake } +func (c *FakeSubmarinerV1alpha1) Brokers(namespace string) v1alpha1.BrokerInterface { + return &FakeBrokers{c, namespace} +} + func (c *FakeSubmarinerV1alpha1) ServiceDiscoveries(namespace string) v1alpha1.ServiceDiscoveryInterface { return &FakeServiceDiscoveries{c, namespace} } diff --git a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/generated_expansion.go index de74cdc98..e681ab349 100644 --- a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/generated_expansion.go @@ -18,6 +18,8 @@ limitations under the License. package v1alpha1 +type BrokerExpansion interface{} + type ServiceDiscoveryExpansion interface{} type SubmarinerExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/submariner_client.go b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/submariner_client.go index b00bdba67..a2092984e 100644 --- a/pkg/client/clientset/versioned/typed/submariner/v1alpha1/submariner_client.go +++ b/pkg/client/clientset/versioned/typed/submariner/v1alpha1/submariner_client.go @@ -26,6 +26,7 @@ import ( type SubmarinerV1alpha1Interface interface { RESTClient() rest.Interface + BrokersGetter ServiceDiscoveriesGetter SubmarinersGetter } @@ -35,6 +36,10 @@ type SubmarinerV1alpha1Client struct { restClient rest.Interface } +func (c *SubmarinerV1alpha1Client) Brokers(namespace string) BrokerInterface { + return newBrokers(c, namespace) +} + func (c *SubmarinerV1alpha1Client) ServiceDiscoveries(namespace string) ServiceDiscoveryInterface { return newServiceDiscoveries(c, namespace) } diff --git a/pkg/subctl/cmd/deploybroker.go b/pkg/subctl/cmd/deploybroker.go index 82d04ce1e..952d71c84 100644 --- a/pkg/subctl/cmd/deploybroker.go +++ b/pkg/subctl/cmd/deploybroker.go @@ -27,15 +27,18 @@ import ( "github.com/submariner-io/submariner-operator/pkg/discovery/globalnet" "github.com/submariner-io/submariner-operator/pkg/subctl/components" + submarinerv1a1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" "github.com/submariner-io/submariner-operator/pkg/broker" "github.com/submariner-io/submariner-operator/pkg/internal/cli" "github.com/submariner-io/submariner-operator/pkg/subctl/datafile" + "github.com/submariner-io/submariner-operator/pkg/subctl/operator/brokercr" + "github.com/submariner-io/submariner-operator/pkg/subctl/operator/submarinerop" ) var ( ipsecSubmFile string globalnetEnable bool - globalnetCidrRange string + globalnetCIDRRange string defaultGlobalnetClusterSize uint serviceDiscoveryEnabled bool componentArr []string @@ -48,7 +51,7 @@ var validComponents = []string{components.ServiceDiscovery, components.Connectiv func init() { deployBroker.PersistentFlags().BoolVar(&globalnetEnable, "globalnet", false, "enable support for Overlapping CIDRs in connecting clusters (default disabled)") - deployBroker.PersistentFlags().StringVar(&globalnetCidrRange, "globalnet-cidr-range", "169.254.0.0/16", + deployBroker.PersistentFlags().StringVar(&globalnetCIDRRange, "globalnet-cidr-range", "169.254.0.0/16", "GlobalCIDR supernet range for allocating GlobalCIDRs to each cluster") deployBroker.PersistentFlags().UintVar(&defaultGlobalnetClusterSize, "globalnet-cluster-size", 8192, "default cluster size for GlobalCIDR allocated to each cluster (amount of global IPs)") @@ -92,15 +95,32 @@ var deployBroker = &cobra.Command{ } if valid, err := isValidGlobalnetConfig(); !valid { - exitOnError("Invalid GlobalCidr configuration", err) + exitOnError("Invalid GlobalCIDR configuration", err) } config, err := getRestConfig(kubeConfig, kubeContext) exitOnError("The provided kubeconfig is invalid", err) status := cli.NewStatus() - status.Start("Deploying broker") - err = broker.Ensure(config) + + status.Start("Setting up broker RBAC") + err = broker.Ensure(config, false) + status.End(cli.CheckForError(err)) + exitOnError("Error setting up broker RBAC", err) + + status.Start("Deploying the Submariner operator") + err = submarinerop.Ensure(status, config, OperatorNamespace, operatorImage()) status.End(cli.CheckForError(err)) + exitOnError("Error deploying the operator", err) + + status.Start("Deploying the broker") + err = brokercr.Ensure(config, OperatorNamespace, populateBrokerSpec()) + if err == nil { + status.QueueSuccessMessage("The broker has been deployed") + status.End(cli.Success) + } else { + status.QueueFailureMessage("Broker deployment failed") + status.End(cli.Failure) + } exitOnError("Error deploying the broker", err) status.Start(fmt.Sprintf("Creating %s file", brokerDetailsFilename)) @@ -134,7 +154,7 @@ var deployBroker = &cobra.Command{ exitOnError("Error setting up service discovery information", err) - err = broker.CreateGlobalnetConfigMap(config, globalnetEnable, globalnetCidrRange, + err = broker.CreateGlobalnetConfigMap(config, globalnetEnable, globalnetCIDRRange, defaultGlobalnetClusterSize, broker.SubmarinerBrokerNamespace) exitOnError("Error creating globalCIDR configmap on Broker", err) @@ -166,9 +186,20 @@ func isValidGlobalnetConfig() (bool, error) { if !globalnetEnable { return true, nil } - defaultGlobalnetClusterSize, err = globalnet.GetValidClusterSize(globalnetCidrRange, defaultGlobalnetClusterSize) + defaultGlobalnetClusterSize, err = globalnet.GetValidClusterSize(globalnetCIDRRange, defaultGlobalnetClusterSize) if err != nil || defaultGlobalnetClusterSize == 0 { return false, err } return true, err } + +func populateBrokerSpec() submarinerv1a1.BrokerSpec { + brokerSpec := submarinerv1a1.BrokerSpec{ + GlobalnetEnabled: globalnetEnable, + GlobalnetCIDRRange: globalnetCIDRRange, + DefaultGlobalnetClusterSize: defaultGlobalnetClusterSize, + Components: componentArr, + DefaultCustomDomains: defaultCustomDomains, + } + return brokerSpec +} diff --git a/pkg/subctl/operator/brokercr/ensure.go b/pkg/subctl/operator/brokercr/ensure.go new file mode 100644 index 000000000..36a7d26f7 --- /dev/null +++ b/pkg/subctl/operator/brokercr/ensure.go @@ -0,0 +1,74 @@ +/* +© 2021 Red Hat, Inc. and others. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package brokercr + +import ( + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" + + submarinerv1a1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" + submarinerclientset "github.com/submariner-io/submariner-operator/pkg/client/clientset/versioned" +) + +const ( + BrokerName = "submariner-broker" +) + +func Ensure(config *rest.Config, namespace string, brokerSpec submarinerv1a1.BrokerSpec) error { + brokerCR := &submarinerv1a1.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Name: BrokerName, + }, + Spec: brokerSpec, + } + + clientSet, err := submarinerclientset.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + + err = createBroker(clientSet, namespace, brokerCR) + + if err != nil { + return err + } + + return nil +} + +func createBroker(clientSet submarinerclientset.Interface, namespace string, brokerCR *submarinerv1a1.Broker) error { + backoff := wait.Backoff{ + Steps: 10, + Duration: 500 * time.Millisecond, + Factor: 1.2, + Jitter: 1, + } + return wait.ExponentialBackoff(backoff, func() (bool, error) { + _, err := clientSet.SubmarinerV1alpha1().Brokers(namespace).Create(brokerCR) + if errors.IsAlreadyExists(err) { + // We can’t always handle existing resources, and we want to overwrite them anyway, so delete them + err := clientSet.SubmarinerV1alpha1().Brokers(namespace).Delete(brokerCR.Name, &metav1.DeleteOptions{}) + return false, err + } else { + return err == nil, err + } + }) +} diff --git a/pkg/subctl/operator/common/embeddedyamls/generators/yamls2go.go b/pkg/subctl/operator/common/embeddedyamls/generators/yamls2go.go index d867cfd72..549b60ca6 100644 --- a/pkg/subctl/operator/common/embeddedyamls/generators/yamls2go.go +++ b/pkg/subctl/operator/common/embeddedyamls/generators/yamls2go.go @@ -26,6 +26,7 @@ import ( ) var files = []string{ + "deploy/crds/submariner.io_brokers.yaml", "deploy/crds/submariner.io_submariners.yaml", "deploy/crds/submariner.io_servicediscoveries.yaml", "deploy/submariner/crds/submariner.io_clusters.yaml", diff --git a/pkg/subctl/operator/submarinerop/crds/ensure.go b/pkg/subctl/operator/submarinerop/crds/ensure.go index a66903e04..1ab1ff750 100644 --- a/pkg/subctl/operator/submarinerop/crds/ensure.go +++ b/pkg/subctl/operator/submarinerop/crds/ensure.go @@ -31,8 +31,13 @@ func Ensure(restConfig *rest.Config) (bool, error) { return false, err } - // Attempt to update or create the CRD definition + // Attempt to update or create the CRD definitions // TODO(majopela): In the future we may want to report when we have updated the existing // CRD definition with new versions - return utils.CreateOrUpdateEmbeddedCRD(crdUpdater, embeddedyamls.Deploy_crds_submariner_io_submariners_yaml) + submarinerCreated, err := utils.CreateOrUpdateEmbeddedCRD(crdUpdater, embeddedyamls.Deploy_crds_submariner_io_submariners_yaml) + if err != nil { + return false, err + } + brokerCreated, err := utils.CreateOrUpdateEmbeddedCRD(crdUpdater, embeddedyamls.Deploy_crds_submariner_io_brokers_yaml) + return submarinerCreated || brokerCreated, err }