VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC Initial virtual kubelet provider for VMware VIC. This provider currently handles creating and starting of a pod VM via the VIC portlayer and persona server. Image store handling via the VIC persona server. This provider currently requires the feature/wolfpack branch of VIC. * Added pod stop and delete. Also added node capacity. Added the ability to stop and delete pod VMs via VIC. Also retrieve node capacity information from the VCH. * Cleanup and readme file Some file clean up and added a Readme.md markdown file for the VIC provider. * Cleaned up errors, added function comments, moved operation code 1. Cleaned up error handling. Set standard for creating errors. 2. Added method prototype comments for all interface functions. 3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder. * Add mocking code and unit tests for podcache, podcreator, and podstarter Used the unit test framework used in VIC to handle assertions in the provider's unit test. Mocking code generated using OSS project mockery, which is compatible with the testify assertion framework. * Vendored packages for the VIC provider Requires feature/wolfpack branch of VIC and a few specific commit sha of projects used within VIC. * Implementation of POD Stopper and Deleter unit tests (#4) * Updated files for initial PR
2018-06-04 15:41:32 -07:00
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions
--- a/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac.go
+++ b/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac.go
@@ -0,0 +1,423 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rbac
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25"
+	"github.com/vmware/govmomi/vim25/methods"
+	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/vmware/vic/pkg/errors"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+const (
+	VCenter = iota
+	DatacenterReadOnly
+	Datacenter
+	Cluster
+	DatastoreFolder
+	Datastore
+	VSANDatastore
+	Network
+	Endpoint
+)
+
+type NameToRef map[string]types.ManagedObjectReference
+
+type AuthzManager struct {
+	authzManager *object.AuthorizationManager
+	client       *vim25.Client
+	resources    map[int8]*Resource
+	TargetRoles  []types.AuthorizationRole
+	RolePrefix   string
+	Principal    string
+	Config       *Config
+}
+
+type Resource struct {
+	Type      int8
+	Propagate bool
+	Role      types.AuthorizationRole
+}
+
+type Config struct {
+	Resources []Resource
+}
+
+type PermissionList []types.Permission
+
+type ResourcePermission struct {
+	RType      int8
+	Reference  types.ManagedObjectReference
+	Permission types.Permission
+}
+
+func NewAuthzManager(ctx context.Context, client *vim25.Client) *AuthzManager {
+	authManager := object.NewAuthorizationManager(client)
+	mgr := &AuthzManager{
+		client:       client,
+		authzManager: authManager,
+	}
+	return mgr
+}
+
+func (am *AuthzManager) InitConfig(principal string, rolePrefix string, config *Config) {
+	am.Principal = principal
+	am.RolePrefix = rolePrefix
+	am.Config = config
+	am.initTargetRoles()
+	am.initResourceMap()
+}
+
+func (am *AuthzManager) CreateRoles(ctx context.Context) (int, error) {
+	return am.createOrRepairRoles(ctx)
+}
+
+func (am *AuthzManager) DeleteRoles(ctx context.Context) (int, error) {
+	return am.deleteRoles(ctx)
+}
+
+func (am *AuthzManager) RoleList(ctx context.Context) (object.AuthorizationRoleList, error) {
+	return am.getRoleList(ctx)
+}
+
+func (am *AuthzManager) IsPrincipalAnAdministrator(ctx context.Context) (bool, error) {
+	// Check if the principal belongs to the Administrators group
+	res, err := am.PrincipalBelongsToGroup(ctx, "Administrators")
+	if err != nil {
+		return false, err
+	}
+
+	if res {
+		return res, nil
+	}
+
+	// Check if the principal has an Admin Role
+	res, err = am.PrincipalHasRole(ctx, "Admin")
+	if err != nil {
+		return false, err
+	}
+
+	return res, nil
+}
+
+func (am *AuthzManager) PrincipalBelongsToGroup(ctx context.Context, group string) (bool, error) {
+	op := trace.FromContext(ctx, "PrincipalBelongsToGroup")
+
+	ref := *am.client.ServiceContent.UserDirectory
+
+	components := strings.Split(am.Principal, "@")
+	var domain string
+	name := components[0]
+	if len(components) < 2 {
+		domain = ""
+	} else {
+		domain = components[1]
+	}
+
+	req := types.RetrieveUserGroups{
+		This:           ref,
+		Domain:         domain,
+		SearchStr:      name,
+		ExactMatch:     true,
+		BelongsToGroup: group,
+		FindUsers:      true,
+		FindGroups:     false,
+	}
+
+	results, err := methods.RetrieveUserGroups(ctx, am.client, &req)
+
+	// This is to work around a bug in vSphere, when AD is added to
+	// the identity source list, the API returns Object Not Found,
+	// In this case, we ignore the error and return false (BUG: 2037706)
+	if err != nil && (isNotSupportedError(ctx, err) || isNotFoundError(ctx, err)) {
+		op.Debugf("Received Error (%s) from PrincipalBelongsToGroup(), could not verify user %s is not a member of the Administrators group", err.Error(), am.Principal)
+		op.Warnf("If ops-user (%s) belongs to the Administrators group, permissions on some resources might have been restricted", am.Principal)
+		return false, nil
+	}
+
+	if err != nil {
+		op.Debugf("Error from PrincipalBelongsToGroup: %s", err.Error())
+		return false, err
+	}
+
+	if len(results.Returnval) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func (am *AuthzManager) PrincipalHasRole(ctx context.Context, roleName string) (bool, error) {
+	// Build expected representation of the ops-user
+	principal := strings.ToLower(am.Principal)
+
+	// Get role id for admin Role
+	roleList, err := am.RoleList(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	role := roleList.ByName(roleName)
+
+	allPerms, err := am.authzManager.RetrieveAllPermissions(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	for _, perm := range allPerms {
+		if perm.RoleId != role.RoleId {
+			continue
+		}
+
+		fPrincipal := am.formatPrincipal(perm.Principal)
+		if fPrincipal == principal {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func (am *AuthzManager) GetPermissions(ctx context.Context,
+	ref types.ManagedObjectReference) ([]types.Permission, error) {
+	// Get current Permissions
+	return am.authzManager.RetrieveEntityPermissions(ctx, ref, false)
+}
+
+func (am *AuthzManager) AddPermission(ctx context.Context, ref types.ManagedObjectReference, resourceType int8, isGroup bool) (*ResourcePermission, error) {
+
+	resource := am.getResource(resourceType)
+	if resource == nil {
+		return nil, fmt.Errorf("cannot find resource of type %d", resourceType)
+	}
+
+	// Collect the new roles, possibly cache the result in the Authz manager
+	roleList, err := am.getRoleList(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Locate target role
+	role := roleList.ByName(am.getRoleName(resource))
+	if role == nil {
+		return nil, fmt.Errorf("cannot find role: %s", resource.Role.Name)
+	}
+
+	// Get current Permissions
+	permissions, err := am.authzManager.RetrieveEntityPermissions(ctx, ref, false)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, permission := range permissions {
+		if permission.Principal == am.Principal &&
+			permission.RoleId == role.RoleId &&
+			permission.Propagate == resource.Propagate {
+			return nil, nil
+		}
+	}
+
+	// No match found, create new permission
+	permission := types.Permission{
+		Principal: am.Principal,
+		RoleId:    role.RoleId,
+		Propagate: resource.Propagate,
+		Group:     isGroup,
+	}
+
+	permissions = append(permissions, permission)
+
+	if err = am.authzManager.SetEntityPermissions(ctx, ref, permissions); err != nil {
+		return nil, err
+	}
+
+	resourcePermission := &ResourcePermission{
+		Permission: permission,
+		Reference:  ref,
+		RType:      resourceType,
+	}
+
+	return resourcePermission, nil
+}
+
+func (am *AuthzManager) createOrRepairRoles(ctx context.Context) (int, error) {
+	// Get all the existing roles
+	mgr := am.authzManager
+	roleList, err := mgr.RoleList(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	var count int
+	for _, targetRole := range am.TargetRoles {
+		foundRole := roleList.ByName(targetRole.Name)
+		if foundRole != nil {
+			isMod, err := am.checkAndRepairRole(ctx, &targetRole, foundRole)
+			if isMod && err == nil {
+				count++
+			}
+		} else {
+			_, err = mgr.AddRole(ctx, targetRole.Name, targetRole.Privilege)
+			if err == nil {
+				count++
+			}
+		}
+		if err != nil {
+			return count, err
+		}
+	}
+	return count, nil
+}
+
+func (am *AuthzManager) deleteRoles(ctx context.Context) (int, error) {
+	mgr := am.authzManager
+	// Get all the existing roles
+	roleList, err := mgr.RoleList(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	var count int
+	for _, targetRole := range am.TargetRoles {
+		foundRole := roleList.ByName(targetRole.Name)
+		if foundRole != nil {
+			err = mgr.RemoveRole(ctx, foundRole.RoleId, true)
+			if err == nil {
+				count++
+			}
+		}
+	}
+	return count, nil
+}
+
+func (am *AuthzManager) getRoleList(ctx context.Context) (object.AuthorizationRoleList, error) {
+	return am.authzManager.RoleList(ctx)
+}
+
+func (am *AuthzManager) checkAndRepairRole(ctx context.Context, tRole *types.AuthorizationRole, fRole *types.AuthorizationRole) (bool, error) {
+	mgr := am.authzManager
+	// Check that the privileges list in Target Role is a subset of the list in Found role
+	fSet := make(map[string]bool)
+	for _, p := range fRole.Privilege {
+		fSet[p] = true
+	}
+
+	var isModified bool
+	for _, p := range tRole.Privilege {
+		if _, found := fSet[p]; !found {
+			// Privilege not found
+			// Add it to the found Role
+			fRole.Privilege = append(fRole.Privilege, p)
+			isModified = true
+		}
+	}
+
+	if !isModified {
+		return false, nil
+	}
+
+	// Not a subset need to call go-vmomi to set the new privileges
+	err := mgr.UpdateRole(ctx, fRole.RoleId, fRole.Name, fRole.Privilege)
+
+	return true, err
+}
+
+func (am *AuthzManager) initTargetRoles() {
+	count := len(am.Config.Resources)
+	roles := make([]types.AuthorizationRole, 0, count)
+	dSet := make(map[string]bool)
+	for index, resource := range am.Config.Resources {
+		name := am.getRoleName(&am.Config.Resources[index])
+		// Discard duplicates
+		if _, found := dSet[name]; !found {
+			role := new(types.AuthorizationRole)
+			*role = resource.Role
+			role.Name = name
+			dSet[name] = true
+			roles = append(roles, *role)
+		}
+	}
+	am.TargetRoles = roles
+}
+
+func (am *AuthzManager) initResourceMap() {
+	am.resources = make(map[int8]*Resource)
+	for i, resource := range am.Config.Resources {
+		am.resources[resource.Type] = &am.Config.Resources[i]
+	}
+}
+
+func (am *AuthzManager) getResource(resourceType int8) *Resource {
+	resource, ok := am.resources[resourceType]
+	if !ok {
+		panic(errors.Errorf("Cannot find RBAC resource type: %d", resourceType))
+	}
+	return resource
+}
+
+func (am *AuthzManager) formatPrincipal(principal string) string {
+	components := strings.Split(principal, "\\")
+	if len(components) != 2 {
+		return strings.ToLower(principal)
+	}
+	ret := strings.ToLower(components[1]) + "@" + strings.ToLower(components[0])
+	return ret
+}
+
+func (am *AuthzManager) getRoleName(resource *Resource) string {
+	switch resource.Type {
+	case DatacenterReadOnly:
+		return resource.Role.Name
+	default:
+		return am.RolePrefix + resource.Role.Name
+	}
+}
+
+func isNotSupportedError(ctx context.Context, err error) bool {
+	op := trace.FromContext(ctx, "isNotSupportedError")
+
+	if soap.IsSoapFault(err) {
+		vimFault := soap.ToSoapFault(err).VimFault()
+		op.Debugf("Error type: %s", reflect.TypeOf(vimFault))
+
+		_, ok := soap.ToSoapFault(err).VimFault().(types.NotSupported)
+		return ok
+	}
+
+	return false
+}
+
+func isNotFoundError(ctx context.Context, err error) bool {
+	op := trace.FromContext(ctx, "isNotFoundError")
+
+	if soap.IsSoapFault(err) {
+		vimFault := soap.ToSoapFault(err).VimFault()
+		op.Debugf("Error type: %s", reflect.TypeOf(vimFault))
+
+		_, ok := soap.ToSoapFault(err).VimFault().(types.NotFound)
+		return ok
+	}
+
+	return false
+}
--- a/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac_test.go
+++ b/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac_test.go
@@ -0,0 +1,240 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rbac
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/vmware/govmomi/simulator"
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/pkg/vsphere/session"
+	"github.com/vmware/vic/pkg/vsphere/test/env"
+)
+
+var Role1 = types.AuthorizationRole{
+	Name: "vcenter",
+	Privilege: []string{
+		"Datastore.Config",
+	},
+}
+
+var Role2 = types.AuthorizationRole{
+	Name: "datacenter",
+	Privilege: []string{
+		"Datastore.Config",
+		"Datastore.FileManagement",
+		"VirtualMachine.Config.AddNewDisk",
+		"VirtualMachine.Config.AdvancedConfig",
+		"VirtualMachine.Config.RemoveDisk",
+		"VirtualMachine.Inventory.Create",
+		"VirtualMachine.Inventory.Delete",
+	},
+}
+
+var Role3 = types.AuthorizationRole{
+	Name: "cluster",
+	Privilege: []string{
+		"Datastore.AllocateSpace",
+		"Datastore.Browse",
+		"Datastore.Config",
+		"Datastore.DeleteFile",
+		"Datastore.FileManagement",
+		"Host.Config.SystemManagement",
+	},
+}
+
+// Configuration for the ops-user
+var testRBACConfig = Config{
+	Resources: []Resource{
+		{
+			Type:      VCenter,
+			Propagate: false,
+			Role:      Role1,
+		},
+		{
+			Type:      Datacenter,
+			Propagate: true,
+			Role:      Role2,
+		},
+		{
+			Type:      Cluster,
+			Propagate: true,
+			Role:      Role3,
+		},
+	},
+}
+
+var testRolePrefix = "test-role-prefix"
+var testUser = "test-user"
+
+func TestRolesSimulatorVPX(t *testing.T) {
+	ctx := context.Background()
+	m := simulator.VPX()
+	defer m.Remove()
+
+	err := m.Create()
+	require.NoError(t, err, "Cannot create VPX Simulator")
+
+	s := m.Service.NewServer()
+	defer s.Close()
+
+	config := &session.Config{
+		Service:   s.URL.String(),
+		Insecure:  true,
+		Keepalive: time.Duration(5) * time.Minute,
+	}
+
+	sess, err := session.NewSession(config).Connect(ctx)
+	require.NoError(t, err, "Cannot connect to VPX Simulator")
+
+	am := NewAuthzManager(ctx, sess.Vim25())
+	am.InitConfig(testUser, testRolePrefix, &testRBACConfig)
+
+	var testRoleNames = []string{
+		"datacenter",
+		"cluster",
+	}
+
+	var testRolePrivileges = []string{
+		"VirtualMachine.Config.AddNewDisk",
+		"Host.Config.SystemManagement",
+	}
+
+	DoTestRoles(ctx, t, am, testRoleNames, testRolePrivileges)
+}
+
+func TestRolesVCenter(t *testing.T) {
+	ctx := context.Background()
+
+	config := &session.Config{
+		Service:   env.URL(t),
+		Insecure:  true,
+		Keepalive: time.Duration(5) * time.Minute,
+	}
+
+	sess, err := session.NewSession(config).Connect(ctx)
+	if err != nil {
+		t.SkipNow()
+	}
+
+	am := NewAuthzManager(ctx, sess.Vim25())
+	am.InitConfig(testUser, testRolePrefix, &testRBACConfig)
+
+	var testRoleNames = []string{
+		"datacenter",
+		"cluster",
+	}
+
+	var testRolePrivileges = []string{
+		"VirtualMachine.Config.AddNewDisk",
+		"Host.Config.SystemManagement",
+	}
+
+	DoTestRoles(ctx, t, am, testRoleNames, testRolePrivileges)
+}
+
+func TestAdminSimulatorVPX(t *testing.T) {
+	ctx := context.Background()
+	m := simulator.VPX()
+	defer m.Remove()
+
+	err := m.Create()
+	require.NoError(t, err, "Cannot create VPX Simulator")
+
+	s := m.Service.NewServer()
+	defer s.Close()
+
+	config := &session.Config{
+		Service:   s.URL.String(),
+		Insecure:  true,
+		Keepalive: time.Duration(5) * time.Minute,
+	}
+
+	sess, err := session.NewSession(config).Connect(ctx)
+	require.NoError(t, err, "Cannot connect to VPX Simulator")
+
+	am := NewAuthzManager(ctx, sess.Vim25())
+	am.InitConfig("admin", "test-role-prefix", &testRBACConfig)
+
+	// Unfortunately the Sim does not have support for looking up group membership
+	// therefore we can only test the presence of the Admin role
+
+	res, err := am.PrincipalHasRole(ctx, "Admin")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.True(t, res, "User Administrator@vsphere.local should have an Admin role")
+
+	// Negative test, principal does not have that role
+	res, err = am.PrincipalHasRole(ctx, "NoAccess")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User Administrator@vsphere.local should have an NoAccess role")
+
+	// Check regular user
+	am.Principal = "nouser@vshpere.local"
+	res, err = am.PrincipalHasRole(ctx, "Admin")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User nouser@vsphere.local should not have an Admin role")
+}
+
+func TestAdminVCenter(t *testing.T) {
+	ctx := context.Background()
+
+	config := &session.Config{
+		Service:   env.URL(t),
+		Insecure:  true,
+		Keepalive: time.Duration(5) * time.Minute,
+	}
+
+	sess, err := session.NewSession(config).Connect(ctx)
+	if err != nil {
+		t.SkipNow()
+	}
+
+	am := NewAuthzManager(ctx, sess.Vim25())
+	am.InitConfig("Administrator@vsphere.local", "test-role-prefix", &testRBACConfig)
+
+	res, err := am.PrincipalBelongsToGroup(ctx, "Administrators")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.True(t, res, "User Administrator@vsphere.local should be a member of Administrators")
+
+	res, err = am.PrincipalHasRole(ctx, "Admin")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.True(t, res, "User Administrator@vsphere.local should have an Admin role")
+
+	// Negative test, principal does not belong
+	res, err = am.PrincipalBelongsToGroup(ctx, "TestUsers")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User Administrator@vsphere.local should not be a member of TestUsers")
+
+	// Negative test, principal does not have that role
+	res, err = am.PrincipalHasRole(ctx, "NoAccess")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User Administrator@vsphere.local should have an NoAccess role")
+
+	// Check regular user
+	am.Principal = "nouser@vshpere.local"
+	res, err = am.PrincipalHasRole(ctx, "Admin")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User nouser@vsphere.local should not have an Admin role")
+
+	// Check regular user
+	am.Principal = "nouser"
+	res, err = am.PrincipalHasRole(ctx, "Admin")
+	require.NoError(t, err, "Failed to verify Admin Privileges")
+	require.False(t, res, "User nouser@vsphere.local should not have an Admin role")
+}
--- a/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac_test_util.go
+++ b/vendor/github.com/vmware/vic/pkg/vsphere/rbac/rbac_test_util.go
@@ -0,0 +1,120 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rbac
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+func DoTestRoles(ctx context.Context, t *testing.T, am *AuthzManager, testRoleNames []string, testRolePrivileges []string) {
+	var roleCount = len(am.TargetRoles)
+	count := InitRoles(ctx, t, am)
+
+	defer Cleanup(ctx, t, am, true)
+	require.Equal(t, roleCount, count, "Incorrect number of roles: expected %d, actual %d", roleCount, count)
+
+	// Test correct role validation, it should return 0
+	roleCount = 0
+	count, err := am.createOrRepairRoles(ctx)
+	require.NoError(t, err, "Failed to create roles")
+	require.Equal(t, roleCount, count, "Incorrect number of roles: expected %d, actual %d", roleCount, count)
+
+	// Remove two Privileges from two roles
+	roles, err := am.getRoleList(ctx)
+	fmt.Println(err)
+	fmt.Println(roles)
+
+	for i, name := range testRoleNames {
+		testRoleNames[i] = am.RolePrefix + name
+	}
+
+	for _, role := range roles {
+		if role.Name == testRoleNames[0] {
+			removePrivilege(&role, testRolePrivileges[0])
+			am.authzManager.UpdateRole(ctx, role.RoleId, role.Name, role.Privilege)
+		}
+		if role.Name == testRoleNames[1] {
+			removePrivilege(&role, testRolePrivileges[1])
+			am.authzManager.UpdateRole(ctx, role.RoleId, role.Name, role.Privilege)
+		}
+	}
+
+	// Test
+	roleCount = 2
+	count, err = am.createOrRepairRoles(ctx)
+	require.NoError(t, err, "Failed to repair roles 1")
+	require.Equal(t, roleCount, count, "Incorrect number of roles: expected %d, actual %d", roleCount, count)
+
+	// Test correct role validation, it should return 0
+	roleCount = 0
+	count, err = am.createOrRepairRoles(ctx)
+	require.NoError(t, err, "Failed to repair roles 2")
+	require.Equal(t, roleCount, count, "Incorrect number of roles: expected %d, actual %d", roleCount, count)
+}
+
+func VerifyResourcePermissions(ctx context.Context, t *testing.T, am *AuthzManager, retPerms []ResourcePermission) {
+	for _, retPerm := range retPerms {
+
+		// Validate returned permission against the configured permission
+		configPerm := am.getResource(retPerm.RType)
+		require.Equal(t, am.Principal, retPerm.Permission.Principal)
+		require.Equal(t, configPerm.Propagate, retPerm.Permission.Propagate)
+
+		actPerms, err := am.GetPermissions(ctx, retPerm.Reference)
+		require.NoError(t, err)
+
+		for _, actPerm := range actPerms {
+			if actPerm.Principal != am.Principal {
+				continue
+			}
+			// RoleId must be the same
+			require.Equal(t, retPerm.Permission.RoleId, actPerm.RoleId)
+		}
+	}
+}
+
+func InitRoles(ctx context.Context, t *testing.T, am *AuthzManager) int {
+	Cleanup(ctx, t, am, false)
+
+	count, err := am.createOrRepairRoles(ctx)
+	require.NoError(t, err, "Failed to initialize Roles")
+
+	return count
+}
+
+func Cleanup(ctx context.Context, t *testing.T, am *AuthzManager, checkCount bool) {
+	var roleCount = len(am.TargetRoles)
+	count, err := am.deleteRoles(ctx)
+	require.NoError(t, err, "Failed to delete roles")
+
+	if checkCount && count != roleCount {
+		t.Fatalf("Incorrect number of roles: expcted %d, actual %d", roleCount, count)
+	}
+}
+
+func removePrivilege(role *types.AuthorizationRole, privilege string) {
+	for i, priv := range role.Privilege {
+		if priv == privilege {
+			role.Privilege = append(role.Privilege[:i], role.Privilege[i+1:]...)
+			return
+		}
+	}
+}