Skip to content

Commit

Permalink
Add unit tests for ROKS support in Calico RA handler
Browse files Browse the repository at this point in the history 
Signed-off-by: Tom Pantelis <[email protected]>
  • Loading branch information
@tpantelis
tpantelis committed Feb 7, 2024
1 parent 61431aa commit be9f06f
Show file tree
Hide file tree
Showing 3 changed files with 154 additions and 88 deletions.
104 changes: 97 additions & 7 deletions pkg/routeagent_driver/handlers/calico/calico_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,31 @@ limitations under the License.
package calico_test

import (
"context"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
calicoapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
calicocs "github.com/projectcalico/api/pkg/client/clientset_generated/clientset"
calicocsfake "github.com/projectcalico/api/pkg/client/clientset_generated/clientset/fake"
"github.com/submariner-io/admiral/pkg/fake"
"github.com/submariner-io/admiral/pkg/log/kzerolog"
"github.com/submariner-io/submariner/pkg/event"
eventtesting "github.com/submariner-io/submariner/pkg/event/testing"
"github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/calico"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fakek8s "k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
)

type testDriver struct {
*eventtesting.ControllerSupport
handler event.Handler
k8sClient *fakek8s.Clientset
handler event.Handler
k8sClient *fakek8s.Clientset
calicoClient *calicocsfake.Clientset
}

func newTestDriver() *testDriver {
Expand All @@ -44,16 +53,18 @@ func newTestDriver() *testDriver {

BeforeEach(func() {
t.k8sClient = fakek8s.NewSimpleClientset()

t.calicoClient = calicocsfake.NewSimpleClientset()
fake.AddDeleteCollectionReactor(&t.calicoClient.Fake)

calico.NewClient = func(_ *rest.Config) (calicocs.Interface, error) {
return t.calicoClient, nil
}
})

return t
}

func (t *testDriver) Start(handler event.Handler) {
t.handler = handler
t.ControllerSupport.Start(handler)
}

var _ = BeforeSuite(func() {
kzerolog.InitK8sLogging()
Expect(calicoapi.AddToScheme(scheme.Scheme)).To(Succeed())
Expand All @@ -63,3 +74,82 @@ func TestCalico(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Calico Suite")
}

func (t *testDriver) Start(handler event.Handler) {
_, err := t.calicoClient.ProjectcalicoV3().IPPools().Create(context.Background(), &calicoapi.IPPool{
ObjectMeta: metav1.ObjectMeta{
Name: calico.DefaultV4IPPoolName,
},
Spec: calicoapi.IPPoolSpec{
IPIPMode: calicoapi.IPIPModeNever,
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())

t.handler = handler
t.ControllerSupport.Start(handler)
}

func (t *testDriver) getIPPoolCIDRs() []string {
list, err := t.calicoClient.ProjectcalicoV3().IPPools().List(context.Background(), metav1.ListOptions{
LabelSelector: calico.SubmarinerIPPool + "=true",
})
Expect(err).To(Succeed())

cidrs := make([]string, len(list.Items))
for i := range list.Items {
cidrs[i] = list.Items[i].Spec.CIDR
}

return cidrs
}

func (t *testDriver) ensureNoIPPools(subnets ...string) {
Consistently(func() []string {
return t.getIPPoolCIDRs()
}).ShouldNot(ContainElements(toAny(subnets)...))
}

func (t *testDriver) ensureIPPools(subnets ...string) {
Consistently(func() []string {
return t.getIPPoolCIDRs()
}).Should(ContainElements(toAny(subnets)...))
}

func (t *testDriver) awaitIPPools(subnets ...string) {
Eventually(func() []string {
return t.getIPPoolCIDRs()
}).Should(ContainElements(toAny(subnets)...))
}

func (t *testDriver) awaitNoIPPools(subnets ...string) {
Eventually(func() []string {
return t.getIPPoolCIDRs()
}).ShouldNot(ContainElements(toAny(subnets)...))
}

func (t *testDriver) getDefaultIPPoolIPIPMode() string {
p, err := t.calicoClient.ProjectcalicoV3().IPPools().Get(context.Background(), calico.DefaultV4IPPoolName, metav1.GetOptions{})
Expect(err).To(Succeed())

return string(p.Spec.IPIPMode)
}

func (t *testDriver) createSubmarinerGwLBService(annotations map[string]string) {
_, err := t.k8sClient.CoreV1().Services(eventtesting.Namespace).Create(context.Background(), &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: calico.GwLBSvcName,
Annotations: annotations,
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())
}

func toAny(s []string) []any {
ia := make([]any, len(s))
for i := range s {
ia[i] = s[i]
}

return ia
}
18 changes: 9 additions & 9 deletions pkg/routeagent_driver/handlers/calico/ippool_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ import (

const (
SubmarinerIPPool = "submariner.io/ippool"
gwLBSvcName = "submariner-gateway"
gwLBSvcROKSAnnotation = "service.kubernetes.io/ibm-load-balancer-cloud-provider-ip-type"
defaultV4IPPoolName = "default-ipv4-ippool"
GwLBSvcName = "submariner-gateway"
GwLBSvcROKSAnnotation = "service.kubernetes.io/ibm-load-balancer-cloud-provider-ip-type"
DefaultV4IPPoolName = "default-ipv4-ippool"
submarinerManagedLabel = "submariner-managed"
submarinerPrevIPIPMode = "submariner-prev-ipipmode"
)
Expand Down Expand Up @@ -214,7 +214,7 @@ func getEndpointSubnetIPPoolName(endpoint *submV1.Endpoint, subnet string) strin

func (h *calicoIPPoolHandler) platformIsROKS() (bool, error) {
// Submariner GW is deployed on ROKS using LB service with specific annotations.
service, err := h.k8sClient.CoreV1().Services(h.namespace).Get(context.TODO(), gwLBSvcName, metav1.GetOptions{})
service, err := h.k8sClient.CoreV1().Services(h.namespace).Get(context.TODO(), GwLBSvcName, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return false, nil
}
Expand All @@ -223,7 +223,7 @@ func (h *calicoIPPoolHandler) platformIsROKS() (bool, error) {
return false, errors.Wrap(err, "error reading gw lb service")
}

return service.GetAnnotations()[gwLBSvcROKSAnnotation] != "", nil
return service.GetAnnotations()[GwLBSvcROKSAnnotation] != "", nil
}

// workaround to address datapath issue with default Calico IPPool configuration for ROKS platform,
Expand All @@ -242,11 +242,11 @@ func (h *calicoIPPoolHandler) updateROKSCalicoCfg() error {

err = util.Update(context.TODO(), h.iPPoolResourceInterface(), &calicoapi.IPPool{
ObjectMeta: metav1.ObjectMeta{
Name: defaultV4IPPoolName,
Name: DefaultV4IPPoolName,
},
}, func(existing *calicoapi.IPPool) (*calicoapi.IPPool, error) {
if existing.Spec.IPIPMode == calicoapi.IPIPModeAlways {
logger.Infof("IPIPMode of %s IPPool already set to Always", defaultV4IPPoolName)
logger.Infof("IPIPMode of %s IPPool already set to Always", DefaultV4IPPoolName)
return existing, nil // no need to update
}

Expand All @@ -262,7 +262,7 @@ func (h *calicoIPPoolHandler) updateROKSCalicoCfg() error {
existing.Labels[submarinerManagedLabel] = "true"
existing.Annotations[submarinerPrevIPIPMode] = string(existing.Spec.IPIPMode)
existing.Spec.IPIPMode = calicoapi.IPIPModeAlways
logger.Infof("Setting IPIPMode of %s IPPool to Always", defaultV4IPPoolName)
logger.Infof("Setting IPIPMode of %s IPPool to Always", DefaultV4IPPoolName)
return existing, nil
})

Expand All @@ -272,7 +272,7 @@ func (h *calicoIPPoolHandler) updateROKSCalicoCfg() error {
func (h *calicoIPPoolHandler) restoreROKSCalicoCfg() error {
err := util.Update(context.TODO(), h.iPPoolResourceInterface(), &calicoapi.IPPool{
ObjectMeta: metav1.ObjectMeta{
Name: defaultV4IPPoolName,
Name: DefaultV4IPPoolName,
},
}, func(existing *calicoapi.IPPool) (*calicoapi.IPPool, error) {
prevIPIPModeCfg := existing.Annotations[submarinerPrevIPIPMode]
Expand Down
120 changes: 48 additions & 72 deletions pkg/routeagent_driver/handlers/calico/ippool_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,131 +24,107 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
calicoapi "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
calicocs "github.com/projectcalico/api/pkg/client/clientset_generated/clientset"
calicocsfake "github.com/projectcalico/api/pkg/client/clientset_generated/clientset/fake"
"github.com/submariner-io/admiral/pkg/fake"
"github.com/submariner-io/submariner/pkg/event/testing"
"github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/calico"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
)

var _ = Describe("IPPool Handler", func() {
t := newTestDriver()

var calicoClient *calicocsfake.Clientset

BeforeEach(func() {
calicoClient = calicocsfake.NewSimpleClientset()
fake.AddDeleteCollectionReactor(&calicoClient.Fake)

calico.NewClient = func(_ *rest.Config) (calicocs.Interface, error) {
return calicoClient, nil
}
})

JustBeforeEach(func() {
t.Start(calico.NewCalicoIPPoolHandler(nil, "", t.k8sClient))
t.Start(calico.NewCalicoIPPoolHandler(nil, testing.Namespace, t.k8sClient))
})

When("remote Endpoints are created and deleted", func() {
It("should create and delete IPPools", func() {
subnets1 := []string{"192.0.2.0/24", "192.0.3.0/24"}
remoteEP1 := t.CreateEndpoint(testing.NewEndpoint("remote-cluster1", "host", subnets1...))
ensureNoIPPools(calicoClient, subnets1...)
t.ensureNoIPPools(subnets1...)

localEP := t.CreateLocalHostEndpoint()
awaitIPPools(calicoClient, subnets1...)
t.awaitIPPools(subnets1...)

// Ensure it handles existing IPPools.
Expect(t.handler.RemoteEndpointCreated(remoteEP1)).To(Succeed())

subnets2 := []string{"192.0.4.0/24"}
remoteEP2 := t.CreateEndpoint(testing.NewEndpoint("remote-cluster1", "host", subnets2...))
awaitIPPools(calicoClient, subnets2...)
t.awaitIPPools(subnets2...)

t.DeleteEndpoint(remoteEP1.Name)
awaitNoIPPools(calicoClient, subnets1...)
t.awaitNoIPPools(subnets1...)

t.DeleteEndpoint(localEP.Name)

t.DeleteEndpoint(remoteEP2.Name)
ensureIPPools(calicoClient, subnets2...)
t.ensureIPPools(subnets2...)
})
})

When("the platform is not ROKS", func() {
Context("because the Submariner GW load balancer is not deployed", func() {
It("should not update the default IPPool's IPIPMode", func() {
Expect(t.getDefaultIPPoolIPIPMode()).Should(Equal(string(calicoapi.IPIPModeNever)))
})
})

Context("because the Submariner GW load balancer does not have the ROKS annotation", func() {
BeforeEach(func() {
t.createSubmarinerGwLBService(map[string]string{})
})

It("should not update the default IPPool's IPIPMode", func() {
Expect(t.getDefaultIPPoolIPIPMode()).Should(Equal(string(calicoapi.IPIPModeNever)))
})
})
})

When("the platform is ROKS", func() {
BeforeEach(func() {
t.createSubmarinerGwLBService(map[string]string{calico.GwLBSvcROKSAnnotation: "foo"})
})

It("should update the default IPPool's IPIPMode to Always", func() {
Expect(t.getDefaultIPPoolIPIPMode()).Should(Equal(string(calicoapi.IPIPModeAlways)))
})
})

Context("on Uninstall", func() {
It("should delete all IPPools", func() {
_, err := calicoClient.ProjectcalicoV3().IPPools().Create(context.Background(), &calicoapi.IPPool{
BeforeEach(func() {
t.createSubmarinerGwLBService(map[string]string{calico.GwLBSvcROKSAnnotation: "foo"})

_, err := t.calicoClient.ProjectcalicoV3().IPPools().Create(context.Background(), &calicoapi.IPPool{
ObjectMeta: metav1.ObjectMeta{
Name: "pool1",
Labels: map[string]string{calico.SubmarinerIPPool: "true"},
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())

_, err = calicoClient.ProjectcalicoV3().IPPools().Create(context.Background(), &calicoapi.IPPool{
_, err = t.calicoClient.ProjectcalicoV3().IPPools().Create(context.Background(), &calicoapi.IPPool{
ObjectMeta: metav1.ObjectMeta{
Name: "pool2",
Labels: map[string]string{calico.SubmarinerIPPool: "true"},
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())
})

JustBeforeEach(func() {
Expect(t.handler.Uninstall()).To(Succeed())
})

list, err := calicoClient.ProjectcalicoV3().IPPools().List(context.Background(), metav1.ListOptions{
It("should delete all IPPools", func() {
list, err := t.calicoClient.ProjectcalicoV3().IPPools().List(context.Background(), metav1.ListOptions{
LabelSelector: calico.SubmarinerIPPool + "=true",
})
Expect(err).To(Succeed())
Expect(list.Items).To(BeEmpty())
})
})
})

func getIPPoolCIDRs(client calicocs.Interface) []string {
list, err := client.ProjectcalicoV3().IPPools().List(context.Background(), metav1.ListOptions{
LabelSelector: calico.SubmarinerIPPool + "=true",
It("should reset the default IPPool's IPIPMode", func() {
Expect(t.getDefaultIPPoolIPIPMode()).Should(Equal(string(calicoapi.IPIPModeNever)))
})
})
Expect(err).To(Succeed())

cidrs := make([]string, len(list.Items))
for i := range list.Items {
cidrs[i] = list.Items[i].Spec.CIDR
}

return cidrs
}

func ensureNoIPPools(client calicocs.Interface, subnets ...string) {
Consistently(func() []string {
return getIPPoolCIDRs(client)
}).ShouldNot(ContainElements(toAny(subnets)...))
}

func ensureIPPools(client calicocs.Interface, subnets ...string) {
Consistently(func() []string {
return getIPPoolCIDRs(client)
}).Should(ContainElements(toAny(subnets)...))
}

func awaitIPPools(client calicocs.Interface, subnets ...string) {
Eventually(func() []string {
return getIPPoolCIDRs(client)
}).Should(ContainElements(toAny(subnets)...))
}

func awaitNoIPPools(client calicocs.Interface, subnets ...string) {
Eventually(func() []string {
return getIPPoolCIDRs(client)
}).ShouldNot(ContainElements(toAny(subnets)...))
}

func toAny(s []string) []any {
ia := make([]any, len(s))
for i := range s {
ia[i] = s[i]
}

return ia
}
})

0 comments on commit be9f06f

Please sign in to comment.