Fix the merged CA bug that use 20180601 for the first several years (#265)

* add log

* fix unitest

* Update directory.go

* more log

* more

* fix merge func

* fix

* fox unittest

* clean up code

* Update directory.go

* Update directory.go

* Update directory_test.go

* Update directory.go

Author: yachang

directory/directory.go

@@ -38,8 +38,9 @@ var (
 
 // CompositeAnnotator wraps several annotators, and calls to Annotate() are forwarded to all of them.
 type CompositeAnnotator struct {
-	// latest date of the component annotators.  This is precomputed, and returned by AnnotatorDate()
-	latestDate time.Time
+	// date of CompositeAnnotator is the earliest date of anntators inside this CA. 
+	// It is precomputed, and returned by AnnotatorDate()
+	date       time.Time
 	annotators []api.Annotator
 }
 
@@ -56,11 +57,19 @@ func (ca CompositeAnnotator) Annotate(ip string, ann *api.GeoData) error {
 	return nil
 }
 
+// PrintALl prints all dates inside this CompositeAnnotator
+func (ca CompositeAnnotator) PrintAll() {
+	log.Println("Date of this CA: ", ca.date.Format("20160102"))
+	log.Println("contains anntators with the following dates:")
+	for i := range ca.annotators {
+		log.Println(ca.annotators[i].AnnotatorDate().Format("20160102"))
+	}
+}
 // AnnotatorDate returns the date of the most recent wrapped annotator.  Most recent is returned
 // as we try to apply the most recent annotators that predate the test we are annotating.  So the
 // most recent of all the annotators is the date that should be compared to the test date.
 func (ca CompositeAnnotator) AnnotatorDate() time.Time {
-	return ca.latestDate
+	return ca.date
 }
 
 // Compute the latest AnnotatorDate() value from a slice of annotators.
@@ -75,6 +84,17 @@ func computeLatestDate(annotators []api.Annotator) time.Time {
 	return t
 }
 
+func computerEarliestDate(annotators []api.Annotator) time.Time {
+	t := time.Now()
+	for i := range annotators {
+		at := annotators[i].AnnotatorDate()
+		if at.Before(t) {
+			t = at
+		}
+	}
+	return t
+}
+
 // String creates a string representation of the CA.
 // Base annotators will appear as [YYYYMMDD], and composite annotators as (A1A2), e.g.,
 // ([20100102]([20110304][20120506]))
@@ -95,7 +115,7 @@ func NewCompositeAnnotator(annotators []api.Annotator) api.Annotator {
 	if annotators == nil {
 		return nil
 	}
-	ca := CompositeAnnotator{latestDate: computeLatestDate(annotators), annotators: annotators}
+	ca := CompositeAnnotator{date: computerEarliestDate(annotators), annotators: annotators}
 	return ca
 }
 
@@ -157,7 +177,7 @@ func advance(lists [][]api.Annotator) ([][]api.Annotator, bool) {
 
 // MergeAnnotators merges multiple lists of annotators, and returns a list of CompositeAnnotators.
 // Result will include a separate CompositeAnnotator for each unique date in any list, and each
-// CA will include the most recent annotator from each list, prior to or equal to the CA date.
+// CA will include annotator of different types, that was the earlist available one after the CA date.
 func MergeAnnotators(lists ...[]api.Annotator) []api.Annotator {
 	listCount := len(lists)
 	if listCount == 0 {
@@ -229,3 +249,11 @@ func (d *Directory) lastEarlierThan(date time.Time) api.Annotator {
 	}
 	return d.annotators[index-1]
 }
+
+func (d *Directory) PrintAll() {
+	log.Println("Here are all datasets in dir currently:")
+	for _, ann := range d.annotators {
+		ann.(CompositeAnnotator).PrintAll()
+	}
+	log.Println("end of dir dataset list")
+}

directory/directory_test.go

@@ -55,6 +55,7 @@ func TestBuild(t *testing.T) {
 	}{
 		{"20170101", "20140608"},
 		{"20110101", "20100124"},
+		{"20180501", "20180408"},
 		// TODO: Add test cases.
 	}
 	for _, tt := range tests {
@@ -115,6 +116,15 @@ func TestMergeAnnotators(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			got := directory.MergeAnnotators(tt.lists[0], tt.lists[1])
 			// This is just a hack to allow us to create a useful signature.
+			expectedDate := make([]string, 3)
+			expectedDate[0] = "[20100101]"
+			expectedDate[1] = "[20100203]"
+			expectedDate[2] = "[20110101]"
+			for i, ann := range got {
+				if expectedDate[i] != ann.AnnotatorDate().Format("[20060102]") {
+					t.Errorf("Expect AnnotateDate %s got %s", expectedDate[i], ann.AnnotatorDate().Format("[20060102]"))
+				}
+			}
 			gotString := directory.NewCompositeAnnotator(got).(directory.CompositeAnnotator).String()
 			if gotString != tt.want {
 				t.Errorf("MergeAnnotators() =\n %v want:\n %v", gotString, tt.want)

manager/manager.go

@@ -188,18 +188,14 @@ func (bldr *listBuilder) build() []api.Annotator {
 
 	// merge the legacy V4 & V6 annotators
 	legacy := mergeV4V6(bldr.legacyV4.Fetch(), bldr.legacyV6.Fetch(), "legacy")
-
 	// Now append the Geolite2 annotators
 	g2 := directory.SortSlice(bldr.geolite2.Fetch())
-
 	geo := make([]api.Annotator, 0, len(g2)+len(legacy))
 	geo = append(geo, legacy...)
 	geo = append(geo, g2...)
-
 	// here we have all the geo annotators in the ordered list.
 	// now merge the ASN V4 & V6 annotators
 	asn := mergeV4V6(bldr.asnV4.Fetch(), bldr.asnV6.Fetch(), "ASN")
-
 	// and now we need to create the composite annotators. First list is the
 	// geo annotators, the second is the ASN
 	combo := directory.MergeAnnotators(geo, asn)
@@ -217,7 +213,6 @@ func (bldr *listBuilder) build() []api.Annotator {
 func mergeV4V6(v4Annotators, v6Annotators []api.Annotator, discriminator string) []api.Annotator {
 	v4 := directory.SortSlice(v4Annotators)
 	v6 := directory.SortSlice(v6Annotators)
-
 	var merged []api.Annotator
 	if len(v4)*len(v6) < 1 {
 		log.Printf("empty v4 or v6 annotator list for %s data, skipping", discriminator)

manager/manager_test.go

@@ -54,7 +54,7 @@ func TestInitDataset(t *testing.T) {
 			`{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":{"Systems":[{"ASNs":[23969]}]}}`},
 		// This request needs a geolite2 dataset
 		{"1.9.128.0", "1512086400",
-			`{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"05","city":"Pantai","latitude":2.787,"longitude":101.995},"Network":{"Systems":[{"ASNs":[4788]}]}}`},
+			`{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"14","city":"Kuala Lumpur","postal_code":"50586","latitude":3.167,"longitude":101.7},"Network":{"Systems":[{"ASNs":[4788]}]}}`},
 		// This request needs the latest dataset in the memory.
 		{"1.22.128.0", "1544400000",
 			`{"Geo":{"continent_code":"AS","country_code":"IN","country_name":"India","region":"HR","city":"Faridabad","latitude":28.4333,"longitude":77.3167},"Network":{"Systems":[{"ASNs":[45528]}]}}`},