Generate LogoSearch index, deal with bad images, better paging

AwesomeLogos · Apr 4, 2024 · 9385f53 · 9385f53
1 parent c6d40e8
commit 9385f53
Show file tree

Hide file tree

Showing 16 changed files with 654 additions and 37 deletions.
diff --git a/bimiHandler.go b/bimiHandler.go
@@ -22,3 +22,53 @@ func bimiHandler(w http.ResponseWriter, r *http.Request) {
 		"Title":  fmt.Sprintf("BIMI Logo for %s", domain),
 	})
 }
+
+func refreshHandler(w http.ResponseWriter, r *http.Request) {
+
+	requestedDomain := r.PathValue("domain")
+	if requestedDomain == "" {
+		http.Error(w, "You must enter a domain", http.StatusBadRequest)
+		return
+	}
+
+	domain, domainErr := getDomain(requestedDomain)
+	if domainErr != nil {
+		http.Error(w, "Unable to get domain", http.StatusNotFound)
+		return
+	}
+
+	bimi, bimiErr := lookupBimi(requestedDomain)
+
+	runTemplate(w, r, "_bimi/index.tmpl", map[string]any{
+		"Bimi":   bimi,
+		"Domain": domain,
+		"Err":    bimiErr,
+		"Title":  fmt.Sprintf("BIMI Logo for %s", domain),
+	})
+}
+
+func sourceHandler(w http.ResponseWriter, r *http.Request) {
+
+	requestedDomain := r.PathValue("domain")
+	if requestedDomain == "" {
+		http.Redirect(w, r, "/?err=You+must+enter+a+domain", http.StatusSeeOther)
+		return
+	}
+
+	domain, domainErr := getDomain(requestedDomain)
+	if domainErr != nil {
+		http.Error(w, "Unable to get domain", http.StatusNotFound)
+		return
+	}
+
+	contentType, body, fetchErr := fetchImgURL(domain.Imgurl.String)
+
+	runTemplate(w, r, "_bimi/source.tmpl", map[string]any{
+		"ContentType": contentType,
+		"Domain":      requestedDomain,
+		"Err":         fetchErr,
+		"Formatted":   body, //LATER: pretty print the XML
+		"Raw":         body,
+		"Title":       fmt.Sprintf("BIMI Logo for %s", domain),
+	})
+}
diff --git a/bulkLoader.go b/bulkLoader.go
@@ -43,9 +43,6 @@ func bulkLoader(filenames []string) {
 		scanner := bufio.NewScanner(bufio.NewReaderSize(file, 100*1024))
 		for scanner.Scan() {
 			domainChan <- scanner.Text()
-			if scanner.Text() == "fileformat.info" {
-				fmt.Fprintf(os.Stderr, "INFO: skipping fileformat.info\n")
-			}
 			count++
 			if count%1000 == 0 {
 				fmt.Fprintf(os.Stderr, ".")

diff --git a/database.go b/database.go
@@ -28,29 +28,26 @@ func connect() (*pgx.Conn, error) {
 	return conn, nil
 }
 
-func upsertDomain(domain, imgurl string) error {
+func countDomains() (int64, error) {
 
 	conn, connErr := connect()
 	if connErr != nil {
 		// already logged
-		return connErr
+		return 0, connErr
 	}
 	defer conn.Close(context.Background())
 
 	queries := generated.New(conn)
-	_, queryErr := queries.UpsertDomain(context.Background(), generated.UpsertDomainParams{
-		Domain: domain,
-		Imgurl: pgtype.Text{String: imgurl, Valid: true},
-	})
+	row, queryErr := queries.CountDomains(context.Background())
 	if queryErr != nil {
-		logger.Error("Unable to insert into database", "err", queryErr)
-		return queryErr
+		logger.Error("Unable to count", "err", queryErr)
+		return 0, queryErr
 	}
 
-	return nil
+	return row, nil
 }
 
-func countDomains() (int64, error) {
+func countInvalidDomains() (int64, error) {
 
 	conn, connErr := connect()
 	if connErr != nil {
@@ -60,16 +57,54 @@ func countDomains() (int64, error) {
 	defer conn.Close(context.Background())
 
 	queries := generated.New(conn)
-	row, queryErr := queries.CountDomains(context.Background())
+	row, queryErr := queries.CountInvalidDomains(context.Background())
+	if queryErr != nil {
+		logger.Error("Unable to count", "err", queryErr)
+		return 0, queryErr
+	}
+
+	return row, nil
+}
+
+func countUnvalidatedDomains() (int64, error) {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return 0, connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	row, queryErr := queries.CountUnvalidatedDomains(context.Background())
 	if queryErr != nil {
-		logger.Error("Unable count rows", "err", queryErr)
+		logger.Error("Unable to count", "err", queryErr)
 		return 0, queryErr
 	}
 
 	return row, nil
 }
 
-func listDomains(limit int32, offset int32) ([]generated.ListDomainsRow, error) {
+func getDomain(domain string) (generated.Domain, error) {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return generated.Domain{}, connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	row, queryErr := queries.GetDomain(context.Background(), domain)
+	if queryErr != nil {
+		logger.Error("Unable to get domain", "err", queryErr)
+		return generated.Domain{}, queryErr
+	}
+
+	return row, nil
+}
+
+func listDomains(limit int32, offset int32) ([]generated.Domain, error) {
 
 	conn, connErr := connect()
 	if connErr != nil {
@@ -90,3 +125,92 @@ func listDomains(limit int32, offset int32) ([]generated.ListDomainsRow, error)
 
 	return rows, nil
 }
+
+func listInvalidDomains(limit int32, offset int32) ([]generated.Domain, error) {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return nil, connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	rows, queryErr := queries.ListInvalidDomains(context.Background(), generated.ListInvalidDomainsParams{
+		Thelimit:  limit,
+		Theoffset: offset,
+	})
+	if queryErr != nil {
+		logger.Error("Unable to insert into database", "err", queryErr)
+		return nil, queryErr
+	}
+
+	return rows, nil
+}
+
+func listUnvalidatedDomains(limit int32, offset int32) ([]generated.Domain, error) {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return nil, connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	rows, queryErr := queries.ListUnvalidatedDomains(context.Background(), generated.ListUnvalidatedDomainsParams{
+		Thelimit:  limit,
+		Theoffset: offset,
+	})
+	if queryErr != nil {
+		logger.Error("Unable to insert into database", "err", queryErr)
+		return nil, queryErr
+	}
+
+	return rows, nil
+}
+
+func updateValidation(domain string, valid bool, reason string) error {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	queryErr := queries.UpdateValidation(context.Background(), generated.UpdateValidationParams{
+		Domain: domain,
+		Valid:  pgtype.Bool{Bool: valid, Valid: true},
+		Reason: pgtype.Text{String: reason, Valid: true},
+	})
+	if queryErr != nil {
+		logger.Error("Unable to update validity in database", "err", queryErr)
+		return queryErr
+	}
+
+	return nil
+}
+
+func upsertDomain(domain, imgurl string) error {
+
+	conn, connErr := connect()
+	if connErr != nil {
+		// already logged
+		return connErr
+	}
+	defer conn.Close(context.Background())
+
+	queries := generated.New(conn)
+	_, queryErr := queries.UpsertDomain(context.Background(), generated.UpsertDomainParams{
+		Domain: domain,
+		Imgurl: pgtype.Text{String: imgurl, Valid: true},
+	})
+	if queryErr != nil {
+		logger.Error("Unable to insert into database", "err", queryErr)
+		return queryErr
+	}
+
+	return nil
+}
diff --git a/db/queries.sql b/db/queries.sql
@@ -1,3 +1,101 @@
+-- name: CountDomains :one
+SELECT 
+        COUNT(*) AS "Count" 
+    FROM 
+        domain
+    WHERE
+        valid IS NULL OR valid = TRUE;
+
+-- name: CountInvalidDomains :one
+SELECT 
+        COUNT(*) AS "Count" 
+    FROM 
+        domain
+    WHERE
+        valid = FALSE;
+
+-- name: CountUnvalidatedDomains :one
+SELECT 
+        COUNT(*) AS "Count" 
+    FROM 
+        domain
+    WHERE
+        valid IS NULL;
+
+-- name: GetDomain :one
+SELECT 
+        * 
+    FROM 
+        domain 
+    WHERE 
+        domain = sqlc.arg(domain);
+
+-- name: ListDomains :many
+SELECT 
+        *
+    FROM 
+        domain 
+    WHERE
+        valid IS NULL OR valid = TRUE
+    ORDER BY 
+        domain 
+    LIMIT 
+        sqlc.arg(theLimit) 
+    OFFSET 
+        sqlc.arg(theOffset);
+
+-- name: ListInvalidDomains :many
+SELECT 
+        *
+    FROM 
+        domain 
+    WHERE
+        valid = FALSE
+    ORDER BY 
+        domain 
+    LIMIT 
+        sqlc.arg(theLimit) 
+    OFFSET 
+        sqlc.arg(theOffset);
+
+-- name: ListUnvalidatedDomains :many
+SELECT 
+        *
+    FROM 
+        domain 
+    WHERE
+        valid IS NULL
+    ORDER BY 
+        domain 
+    LIMIT 
+        sqlc.arg(theLimit) 
+    OFFSET 
+        sqlc.arg(theOffset);
+
+-- name: SearchDomains :many
+SELECT 
+        *
+    FROM 
+        domain 
+    WHERE
+        domain LIKE sqlc.arg(search)
+    ORDER BY 
+        domain 
+    LIMIT 
+        sqlc.arg(theLimit) 
+    OFFSET 
+        sqlc.arg(theOffset);
+
+-- name: UpdateValidation :exec
+UPDATE 
+        domain 
+    SET 
+        valid = sqlc.arg(valid),
+        reason = sqlc.arg(reason),
+        updated = NOW()
+    WHERE 
+        domain = sqlc.arg(domain);
+
 -- name: UpsertDomain :one
 INSERT INTO domain 
         (domain, imgurl)
@@ -8,8 +106,3 @@ INSERT INTO domain
         updated = NOW()
     RETURNING *;
 
--- name: ListDomains :many
-SELECT domain, imgurl FROM domain ORDER BY domain LIMIT sqlc.arg(theLimit) OFFSET sqlc.arg(theOffset);
-
--- name: CountDomains :one
-SELECT COUNT(*) AS "Count" FROM domain;
diff --git a/db/schema.sql b/db/schema.sql
@@ -14,7 +14,9 @@ CREATE TABLE domain (
 	domain VARCHAR(255) NOT NULL PRIMARY KEY,
 	imgurl VARCHAR(1024),
 	headers VARCHAR(1024),
-	imgsvg VARCHAR(32768)
+	imgsvg VARCHAR(32768),
+	valid BOOLEAN,
+	reason VARCHAR(1024)
 );
 
 CREATE INDEX domain_created ON domain (created);