Audit tooling updates

audit/common/Constants.go

@@ -107,6 +107,7 @@ const (
 	StreamProcessingDir = "atlas-stream-processing"
 	TriggersDir         = "triggers"
 	VectorSearchDir     = "atlas-vector-search"
+	AiIntegrationsDir   = "ai-integrations"
 )
 
 var CanonicalLanguages = []string{Bash, C, CPP,

audit/common/GetProductInfo.go

@@ -23,6 +23,7 @@ func GetProductInfo(projectOrSubdir string) ProductInfo {
 }
 
 var SubProductDirs = []string{
+	AiIntegrationsDir,
 	DataFederationDir,
 	OnlineArchiveDir,
 	StreamProcessingDir,
@@ -32,6 +33,11 @@ var SubProductDirs = []string{
 }
 
 var productInfoMap = map[string]ProductInfo{
+	"ai-integrations": {
+		ProductName: Atlas,
+		ProductType: DirIsSubProduct,
+		SubProduct:  VectorSearch,
+	},
 	"atlas-cli": {
 		ProductName: Atlas,
 		ProductType: CollectionIsSubProduct,
@@ -137,6 +143,11 @@ var productInfoMap = map[string]ProductInfo{
 		ProductType: CollectionIsProduct,
 		SubProduct:  "",
 	},
+	"drivers": {
+		ProductName: Drivers,
+		ProductType: CollectionIsProduct,
+		SubProduct:  "",
+	},
 	"entity-framework": {
 		ProductName: EFCoreProvider,
 		ProductType: CollectionIsProduct,

audit/dodec/README.md

@@ -25,6 +25,8 @@ Get counts broken down in the following ways:
   - [By product](src/aggregations/GetProductCategoryCounts.go)
   - [By Atlas sub-product](src/aggregations/GetSubProductCategoryCounts.go)
   - [For one specific category by product](src/aggregations/GetSpecificCategoryByProduct.go)
+  - [For usage examples added in the last month](src/aggregations/FindUsageExamplesForMonth.go)
+  - [For applied usage examples (>300 characters) added in the last week](src/aggregations/FindNewAppliedUsageExamples.go)
 - Complexity counts:
   - [One line usage examples by docs property](src/aggregations/GetOneLineUsageExampleCounts.go)
   - [Minimum, median, maximum character counts for code examples in the collection, and one-liner counts](src/aggregations/GetMinMedianMaxCodeLength.go)

audit/dodec/src/PerformAggregation.go

@@ -5,6 +5,8 @@ import (
 	"dodec/aggregations"
 	"dodec/types"
 	"dodec/utils"
+	"time"
+
 	"go.mongodb.org/mongo-driver/v2/bson"
 	"go.mongodb.org/mongo-driver/v2/mongo"
 )
@@ -36,6 +38,7 @@ func PerformAggregation(db *mongo.Database, ctx context.Context) {
 	}
 	//substringToFindInCodeExamples := "defaultauthdb"
 	//substringToFindInPageURL := "code-examples"
+	monthForReporting := time.November
 
 	for _, collectionName := range collectionNames {
 		//simpleMap = aggregations.GetCategoryCounts(db, collectionName, simpleMap, ctx)
@@ -56,7 +59,8 @@ func PerformAggregation(db *mongo.Database, ctx context.Context) {
 		//pageIdChangesCountMap = aggregations.GetDocsIdsWithRecentActivity(db, collectionName, pageIdChangesCountMap, ctx)
 		//pageIdsWithNodeLangCountMismatch = aggregations.GetPagesWithNodeLangCountMismatch(db, collectionName, pageIdsWithNodeLangCountMismatch, ctx)
 		//pageIdsWithNodeLangCountMismatch = aggregations.FindDocsMissingProduct(db, collectionName, pageIdsWithNodeLangCountMismatch, ctx)
-		productSubProductCounter = aggregations.FindNewAppliedUsageExamples(db, collectionName, productSubProductCounter, ctx)
+		//productSubProductCounter = aggregations.FindNewAppliedUsageExamples(db, collectionName, productSubProductCounter, ctx)
+		productSubProductCounter = aggregations.FindUsageExamplesForMonth(db, collectionName, productSubProductCounter, monthForReporting, ctx)
 		//pageIdsWithNodeLangCountMismatch = aggregations.FindURLContainingString(db, collectionName, pageIdsWithNodeLangCountMismatch, ctx, substringToFindInPageURL)
 	}
 
@@ -79,5 +83,6 @@ func PerformAggregation(db *mongo.Database, ctx context.Context) {
 	//utils.PrintCodeLengthMapToConsole(codeLengthMap)
 	//utils.PrintPageIdChangesCountMap(pageIdChangesCountMap)
 	//utils.PrintPageIdsWithNodeLangCountMismatch(pageIdsWithNodeLangCountMismatch)
-	utils.PrintPageIdNewAppliedUsageExampleCounts(productSubProductCounter)
+	//utils.PrintPageIdNewAppliedUsageExampleCounts(productSubProductCounter)
+	utils.PrintMonthlyUsageExampleCounts(productSubProductCounter, monthForReporting)
 }

audit/dodec/src/aggregations/FindUsageExamplesForMonth.go

@@ -0,0 +1,118 @@
+package aggregations
+
+import (
+	"common"
+	"context"
+	"dodec/types"
+	"dodec/utils"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+)
+
+// FindUsageExamplesForMonth looks for docs pages in Atlas that have had a new usage example added during the target month.
+// We get a count of new usage examples matching this criteria, return the count and the page_id, and
+// track the product and sub-product in the types.NewAppliedUsageExampleCounterByProductSubProduct
+func FindUsageExamplesForMonth(db *mongo.Database, collectionName string, productSubProductCounter types.NewAppliedUsageExampleCounterByProductSubProduct, monthForReporting time.Month, ctx context.Context) types.NewAppliedUsageExampleCounterByProductSubProduct {
+	// Target a specific month (example for November 2025):
+	targetYear := 2025
+	monthStart := time.Date(targetYear, monthForReporting, 1, 0, 0, 0, 0, time.UTC)
+	monthEnd := monthStart.AddDate(0, 1, 0) // First day of next month
+	// Define the aggregation pipeline
+	pipeline := mongo.Pipeline{
+		// Find only page documents where the `nodes` value is not null
+		{{"$match", bson.D{
+			{"_id", bson.D{{"$ne", "summaries"}}},
+			{"nodes", bson.D{{"$ne", nil}}}, // Ensure nodes is not null
+		}}},
+
+		// Unwind the `nodes` value to match on specific node elements
+		{{"$unwind", bson.D{{"path", "$nodes"}}}},
+
+		{{"$match", bson.D{
+			{"$and", bson.A{
+				// Filter for nodes that have been added during the target month
+				bson.D{{"$and", bson.A{
+					bson.D{{"nodes.date_added", bson.D{{"$gte", monthStart}}}},
+					bson.D{{"nodes.date_added", bson.D{{"$lt", monthEnd}}}},
+				}}},
+				// Consider only usage examples
+				bson.D{{"nodes.category", common.UsageExample}},
+			}},
+		}}},
+
+		// First group by Product and SubProduct
+		bson.D{{"$group", bson.D{
+			{"_id", bson.D{
+				{"product", "$product"},
+				{"subProduct", bson.D{{"$ifNull", bson.A{"$sub_product", "None"}}}},
+			}},
+			{"nodesPerProduct", bson.D{{"$push", bson.D{
+				{"_id", "$_id"},     // Preserve original document _id
+				{"nodes", "$nodes"}, // Collect nodes
+			}}}},
+		}}},
+		// Unwind after the first group to regroup by original _id
+		bson.D{{"$unwind", bson.D{{"path", "$nodesPerProduct"}}}},
+		// Regroup by original document _id within each Product and SubProduct
+		bson.D{{"$group", bson.D{
+			{"_id", bson.D{
+				{"product", "$_id.product"},
+				{"subProduct", "$_id.subProduct"},
+				{"documentId", "$nodesPerProduct._id"},
+			}},
+			{"new_applied_usage_examples", bson.D{{"$push", "$nodesPerProduct.nodes"}}},
+			{"count", bson.D{{"$sum", 1}}},
+		}}},
+		// Optionally sort by count in descending order
+		bson.D{{"$sort", bson.D{{"count", -1}}}},
+	}
+	// Execute the aggregation
+	collection := db.Collection(collectionName)
+	cursor, err := collection.Aggregate(ctx, pipeline)
+	if err != nil {
+		println(fmt.Errorf("failed to execute aggregate query: %v", err))
+		return productSubProductCounter
+	}
+	defer cursor.Close(ctx)
+	collectionPagesWithNewAppliedUsageExamples := make([]types.PageIdNewAppliedUsageExamples, 0)
+	for cursor.Next(ctx) {
+		var result types.PageIdNewAppliedUsageExamples
+		if err = cursor.Decode(&result); err != nil {
+			println(fmt.Errorf("failed to decode result document: %v", err))
+			return productSubProductCounter
+		}
+
+		// If a sub-product map for the product does not exist yet, create one
+		if _, ok := productSubProductCounter.ProductSubProductCounts[result.ID.Product]; !ok {
+			productSubProductCounter.ProductSubProductCounts[result.ID.Product] = make(map[string]int)
+		}
+
+		// The docs org would like to see a breakdown of focus areas. For the purpose of reporting this result, I'm arbitrarily
+		// assigning some of the key focus areas as "sub-product" if a page ID contains a substring related to these focus
+		// areas. That makes it easy to report on these things as sub-products even if they're not officially sub-products.
+		resultAdjustedForFocusAreas := utils.GetFocusAreaAsSubProduct(result)
+		if resultAdjustedForFocusAreas.ID.SubProduct != "None" {
+			productSubProductCounter.ProductSubProductCounts[result.ID.Product][resultAdjustedForFocusAreas.ID.SubProduct] += resultAdjustedForFocusAreas.Count
+
+			// Add the adjusted for focus area count to the product accumulator
+			productSubProductCounter.ProductAggregateCount[result.ID.Product] += resultAdjustedForFocusAreas.Count
+		} else {
+			// If the subproduct is "None", just append the original count
+			productSubProductCounter.ProductSubProductCounts[result.ID.Product][result.ID.SubProduct] += result.Count
+			// Add the non-adjusted subproduct count to the product accumulator
+			productSubProductCounter.ProductAggregateCount[result.ID.Product] += result.Count
+		}
+		collectionPagesWithNewAppliedUsageExamples = append(collectionPagesWithNewAppliedUsageExamples, resultAdjustedForFocusAreas)
+	}
+	if err = cursor.Err(); err != nil {
+		println(fmt.Errorf("cursor encountered an error: %v", err))
+		return productSubProductCounter
+	}
+	if collectionPagesWithNewAppliedUsageExamples != nil && len(collectionPagesWithNewAppliedUsageExamples) > 0 {
+		productSubProductCounter.PagesInCollections[collectionName] = collectionPagesWithNewAppliedUsageExamples
+	}
+	return productSubProductCounter
+}

audit/dodec/src/go.mod

@@ -1,24 +1,24 @@
 module dodec
 
-go 1.23.1
+go 1.24.0
 
 require (
 	common v0.0.0
 	github.com/joho/godotenv v1.5.1
-	go.mongodb.org/mongo-driver/v2 v2.2.2
+	go.mongodb.org/mongo-driver/v2 v2.4.0
 )
 
 require (
 	github.com/golang/snappy v1.0.0 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.mongodb.org/mongo-driver v1.17.4 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
-	golang.org/x/sync v0.13.0 // indirect
-	golang.org/x/text v0.24.0 // indirect
+	go.mongodb.org/mongo-driver v1.17.6 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 )
 
 replace common => ../../common

audit/dodec/src/go.sum

@@ -8,6 +8,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
+github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
@@ -19,14 +21,20 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw=
 go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
+go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
+go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.mongodb.org/mongo-driver/v2 v2.2.1 h1:w5xra3yyu/sGrziMzK1D0cRRaH/b7lWCSsoN6+WV6AM=
 go.mongodb.org/mongo-driver/v2 v2.2.1/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps=
 go.mongodb.org/mongo-driver/v2 v2.2.2 h1:9cYuS3fl1Xhqwpfazso10V7BHQD58kCgtzhfAmJYz9c=
 go.mongodb.org/mongo-driver/v2 v2.2.2/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps=
+go.mongodb.org/mongo-driver/v2 v2.4.0 h1:Oq6BmUAAFTzMeh6AonuDlgZMuAuEiUxoAD1koK5MuFo=
+go.mongodb.org/mongo-driver/v2 v2.4.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
 golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -35,6 +43,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
 golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -48,6 +58,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=