-
Notifications
You must be signed in to change notification settings - Fork 430
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* PMM-8489 Fixed topology labels on mongos * Updated mog for go 1.16 * Updated dependency * Updated toolkit dep to 3.x * Updated go.sum for go 1.16 In my local I have go 1.17 so the sum was different and tests were failing. * Fixed mod conflicts * Revert "Merge branch 'PMM-8489_topology_labels_on_mongos'" This reverts commit 8905fce, reversing changes made to 4553084. * PMM-8770 Updated discovery mode Now discovering mode automatically search for all databases and all collections. It might receive a filter list and ignores system collections. * PMM-8770 Added common functions * PMM-8770 New collstats limit * Fixed race condition * Top collector won't be enabled on mongos * Fixes for CR * Renamed collector.topmetric param Co-authored-by: JiriCtvrtka <[email protected]>
- Loading branch information
1 parent
705db98
commit c8360b4
Showing
7 changed files
with
248 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
package exporter | ||
|
||
import ( | ||
"context" | ||
|
||
"github.com/AlekSi/pointer" | ||
"github.com/pkg/errors" | ||
"go.mongodb.org/mongo-driver/bson" | ||
"go.mongodb.org/mongo-driver/bson/primitive" | ||
"go.mongodb.org/mongo-driver/mongo" | ||
"go.mongodb.org/mongo-driver/mongo/options" | ||
) | ||
|
||
var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals | ||
|
||
func listCollections(ctx context.Context, client *mongo.Client, collections []string, database string) ([]string, error) { | ||
filter := bson.D{} // Default=empty -> list all collections | ||
|
||
// if there is a filter with the list of collections we want, create a filter like | ||
// $or: { | ||
// {"$regex": "collection1"}, | ||
// {"$regex": "collection2"}, | ||
// } | ||
if len(collections) > 0 { | ||
matchExpressions := []bson.D{} | ||
|
||
for _, collection := range collections { | ||
matchExpressions = append(matchExpressions, | ||
bson.D{{Key: "name", Value: primitive.Regex{Pattern: collection, Options: "i"}}}) | ||
} | ||
|
||
filter = bson.D{{Key: "$or", Value: matchExpressions}} | ||
} | ||
|
||
databases, err := client.Database(database).ListCollectionNames(ctx, filter) | ||
if err != nil { | ||
return nil, errors.Wrap(err, "cannot get the list of collections for discovery") | ||
} | ||
|
||
return databases, nil | ||
} | ||
|
||
func databases(ctx context.Context, client *mongo.Client, exclude []string) ([]string, error) { | ||
opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)} | ||
filterExpressions := []bson.D{} | ||
for _, dbname := range exclude { | ||
filterExpressions = append(filterExpressions, | ||
bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}}, | ||
) | ||
} | ||
|
||
filter := bson.D{{Key: "$and", Value: filterExpressions}} | ||
|
||
dbNames, err := client.ListDatabaseNames(ctx, filter, opts) | ||
if err != nil { | ||
return nil, errors.Wrap(err, "cannot get the database names list") | ||
} | ||
|
||
return dbNames, nil | ||
} | ||
|
||
func listAllCollections(ctx context.Context, client *mongo.Client, filter []string) (map[string][]string, error) { | ||
namespaces := make(map[string][]string) | ||
// exclude system databases | ||
dbnames, err := databases(ctx, client, systemDBs) | ||
if err != nil { | ||
return nil, errors.Wrap(err, "cannot get the list of all collections in the server") | ||
} | ||
|
||
for _, dbname := range dbnames { | ||
colls, err := listCollections(ctx, client, filter, dbname) | ||
if err != nil { | ||
return nil, errors.Wrapf(err, "cannot list the collections for %q", dbname) | ||
} | ||
namespaces[dbname] = colls | ||
} | ||
|
||
return namespaces, nil | ||
} | ||
|
||
func allCollectionsCount(ctx context.Context, client *mongo.Client, filter []string) (int, error) { | ||
databases, err := databases(ctx, client, systemDBs) | ||
if err != nil { | ||
return 0, errors.Wrap(err, "cannot retrieve the collection names for count collections") | ||
} | ||
|
||
var count int | ||
|
||
for _, dbname := range databases { | ||
colls, err := listCollections(ctx, client, filter, dbname) | ||
if err != nil { | ||
return 0, errors.Wrap(err, "cannot get collections count") | ||
} | ||
count += len(colls) | ||
} | ||
|
||
return count, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
package exporter | ||
|
||
import ( | ||
"context" | ||
"sort" | ||
"testing" | ||
"time" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"go.mongodb.org/mongo-driver/bson" | ||
|
||
"github.com/percona/mongodb_exporter/internal/tu" | ||
) | ||
|
||
func TestListCollections(t *testing.T) { | ||
t.Parallel() | ||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) | ||
defer cancel() | ||
|
||
client := tu.DefaultTestClient(ctx, t) | ||
|
||
databases := []string{"testdb01", "testdb02"} | ||
collections := []string{"col01", "col02", "colxx", "colyy"} | ||
|
||
defer func() { | ||
for _, dbname := range databases { | ||
client.Database(dbname).Drop(ctx) //nolint:errcheck | ||
} | ||
}() | ||
|
||
for _, dbname := range databases { | ||
for _, coll := range collections { | ||
for j := 0; j < 10; j++ { | ||
_, err := client.Database(dbname).Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"}) | ||
assert.NoError(t, err) | ||
} | ||
} | ||
} | ||
|
||
want := []string{"col01", "col02", "colxx"} | ||
collections, err := listCollections(ctx, client, []string{"col0", "colx"}, databases[0]) | ||
sort.Strings(collections) | ||
|
||
assert.NoError(t, err) | ||
assert.Equal(t, want, collections) | ||
|
||
count, err := allCollectionsCount(ctx, client, nil) | ||
assert.NoError(t, err) | ||
assert.True(t, count > 8) | ||
|
||
count, err = allCollectionsCount(ctx, client, []string{"col0", "colx"}) | ||
assert.NoError(t, err) | ||
assert.Equal(t, 6, count) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.