From 36530f4c8995c8d447a6a1e5e526069195701bf8 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sat, 23 Sep 2023 21:43:17 +0200 Subject: [PATCH 1/8] Added see more links to same series and same authors related docs --- internal/controller/document.go | 1 + internal/controller/search.go | 1 + internal/index/bleve.go | 4 ++++ internal/index/bleve_read.go | 21 +++++++++++++++++++ internal/metadata/epub.go | 13 +++++++++--- internal/metadata/metadata.go | 2 ++ .../webserver/embedded/translations/es.yml | 1 + .../webserver/embedded/translations/fr.yml | 1 + .../webserver/embedded/views/document.html | 18 +++++++++++----- .../webserver/embedded/views/results.html | 4 ++-- 10 files changed, 56 insertions(+), 10 deletions(-) diff --git a/internal/controller/document.go b/internal/controller/document.go index f0d8836..30be560 100644 --- a/internal/controller/document.go +++ b/internal/controller/document.go @@ -60,6 +60,7 @@ func Document(c *fiber.Ctx, libraryPath string, sender Sender, idx IdxReader, wo "Lang": lang, "Title": title, "Document": document, + "Authors": strings.Join(document.Authors, ","), "EmailSendingConfigured": emailSendingConfigured, "EmailFrom": sender.From(), "Session": session, diff --git a/internal/controller/search.go b/internal/controller/search.go index 6916ec8..a706398 100644 --- a/internal/controller/search.go +++ b/internal/controller/search.go @@ -70,6 +70,7 @@ func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64) "WordsPerMinute": wordsPerMinute, }, "layout") } + count, err := idx.Count() if err != nil { return fiber.ErrInternalServerError diff --git a/internal/index/bleve.go b/internal/index/bleve.go index a9bf130..f1333f1 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -54,6 +54,10 @@ func Mapping() *mapping.IndexMappingImpl { indexMapping.DefaultMapping.AddFieldMappingsAt("Year", yearFieldMapping) slugFieldMapping := bleve.NewKeywordFieldMapping() indexMapping.DefaultMapping.AddFieldMappingsAt("Slug", slugFieldMapping) + seriesEqFieldMapping := bleve.NewKeywordFieldMapping() + indexMapping.DefaultMapping.AddFieldMappingsAt("SeriesEq", seriesEqFieldMapping) + authorsEqFieldMapping := bleve.NewKeywordFieldMapping() + indexMapping.DefaultMapping.AddFieldMappingsAt("AuthorsEq", authorsEqFieldMapping) return indexMapping } diff --git a/internal/index/bleve_read.go b/internal/index/bleve_read.go index 515a5fb..1283879 100644 --- a/internal/index/bleve_read.go +++ b/internal/index/bleve_read.go @@ -4,11 +4,13 @@ import ( "fmt" "html/template" "math" + "net/url" "path/filepath" "strings" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search/query" + "github.com/gosimple/slug" "github.com/svera/coreander/v3/internal/controller" "github.com/svera/coreander/v3/internal/metadata" ) @@ -23,6 +25,25 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*contr } } + for _, prefix := range []string{"AuthorsEq:", "SeriesEq:"} { + unescaped, err := url.QueryUnescape(strings.TrimSpace(keywords)) + if err != nil { + break + } + if strings.HasPrefix(unescaped, prefix) { + unescaped = strings.Replace(unescaped, prefix, "", 1) + terms := strings.Split(unescaped, ",") + qb := bleve.NewDisjunctionQuery() + for _, term := range terms { + term = strings.ReplaceAll(slug.Make(term), "-", "") + qs := bleve.NewTermQuery(term) + qs.SetField(strings.TrimSuffix(prefix, ":")) + qb.AddQuery(qs) + } + return b.runPaginatedQuery(qb, page, resultsPerPage) + } + } + splitted := strings.Split(strings.TrimSpace(keywords), " ") var ( diff --git a/internal/metadata/epub.go b/internal/metadata/epub.go index 95390f8..ac351d0 100644 --- a/internal/metadata/epub.go +++ b/internal/metadata/epub.go @@ -4,7 +4,7 @@ import ( "archive/zip" "fmt" "html/template" - "io/ioutil" + "io" "log" "path/filepath" "strconv" @@ -14,6 +14,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/disintegration/imaging" "github.com/gofiber/fiber/v2" + "github.com/gosimple/slug" "github.com/microcosm-cc/bluemonday" "github.com/pirmd/epub" ) @@ -30,17 +31,21 @@ func (e EpubReader) Metadata(file string) (Metadata, error) { if len(opf.Metadata.Title) > 0 && len(opf.Metadata.Title[0].Value) > 0 { title = opf.Metadata.Title[0].Value } - var authors []string + var authors, authorsEq []string if len(opf.Metadata.Creator) > 0 { for _, creator := range opf.Metadata.Creator { if creator.Role == "aut" || creator.Role == "" { // Some epub files mistakenly put all authors in a single field instead of using a field for each one. // We want to identify those cases looking for specific separators and then indexing each author properly. names := strings.Split(creator.Value, "&") + namesEq := make([]string, len(names)) + copy(namesEq, names) for i := range names { names[i] = strings.TrimSpace(names[i]) + namesEq[i] = strings.ReplaceAll(slug.Make(namesEq[i]), "-", "") } authors = append(authors, names...) + authorsEq = append(authorsEq, namesEq...) } } } @@ -115,11 +120,13 @@ func (e EpubReader) Metadata(file string) (Metadata, error) { bk = Metadata{ Title: title, Authors: authors, + AuthorsEq: authorsEq, Description: template.HTML(description), Language: language, Year: year, Cover: cover, Series: series, + SeriesEq: strings.ReplaceAll(slug.Make(series), "-", ""), SeriesIndex: seriesIndex, Type: "EPUB", Subjects: subjects, @@ -177,7 +184,7 @@ func words(bookFullPath string) (int, error) { if err != nil { return 0, err } - content, err := ioutil.ReadAll(rc) + content, err := io.ReadAll(rc) if err != nil { return 0, err } diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go index 5ba4a72..b93088b 100644 --- a/internal/metadata/metadata.go +++ b/internal/metadata/metadata.go @@ -12,12 +12,14 @@ type Metadata struct { Slug string Title string Authors []string + AuthorsEq []string Description template.HTML Language string Year string Words float64 Cover string Series string + SeriesEq string SeriesIndex float64 Pages int Type string diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index f018177..27e14e4 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -98,3 +98,4 @@ "Other documents with similar subjects": "Otros documentos de temática similar" "Other documents by": "Otros documentos de" "Other documents in collection \"%s\"": "Otros documentos en la colección \"%s\"" +"See all": "Ver todos" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 33431f8..50f7e10 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -98,3 +98,4 @@ "Other documents with similar subjects": "Autres documents avec des sujets similaires" "Other documents by": "Autres documents par" "Other documents in collection \"%s\"": "Autres documents en collection \"%s\"" +"See all": "Voir tout" diff --git a/internal/webserver/embedded/views/document.html b/internal/webserver/embedded/views/document.html index e3ea621..2d4c290 100644 --- a/internal/webserver/embedded/views/document.html +++ b/internal/webserver/embedded/views/document.html @@ -59,9 +59,7 @@
{{ if ne .Document.Series "" }} {{$seriesTitle := t $lang "Search for more titles belonging to %s" .Document.Series}} -

{{.Document.Series}} {{.Document.SeriesIndex}}

+

{{.Document.Series}} {{.Document.SeriesIndex}}

{{ end }}

{{.Document.Title}} {{.Document.Year}} @@ -111,16 +109,21 @@

{{t $lang "Unknown author"}}

{{ $length := len .SameSeries }} {{ if gt $length 0 }}
-
+

{{t $lang "Other documents in collection \"%s\"" .Document.Series}}

+
+ + {{t $lang "See all" }} +
+ {{template "partials/related" dict "Lang" $lang "Related" .SameSeries}}
{{end}} {{ $length := len .SameAuthors }} {{ if gt $length 0 }}
-
+

{{t $lang "Other documents by"}} {{range $i, $author := $document.Authors}} @@ -128,6 +131,11 @@

{{end}}

+
+ + {{t $lang "See all" }} +
+ {{template "partials/related" dict "Lang" $lang "Related" .SameAuthors}}
{{end}} diff --git a/internal/webserver/embedded/views/results.html b/internal/webserver/embedded/views/results.html index b788529..ea38168 100644 --- a/internal/webserver/embedded/views/results.html +++ b/internal/webserver/embedded/views/results.html @@ -24,7 +24,7 @@ {{ if ne $book.Series "" }} {{$seriesTitle := t $lang "Search for more titles belonging to %s" $book.Series}}

{{$book.Series}} {{$book.SeriesIndex}}

{{ end }}

@@ -42,7 +42,7 @@

{{range $i, $author := $book.Authors}} {{$authorTitle := t $lang "Search for more titles by %s" $author}} - {{$author}}{{if notLast $book.Authors $i}}, {{end}} + {{$author}}{{if notLast $book.Authors $i}}, {{end}} {{end}}
{{else}} From decc678cc0964e0d2c3b88714f3efc88ed3586ba Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sun, 24 Sep 2023 18:47:12 +0200 Subject: [PATCH 2/8] Refactored doc types using embedding --- internal/controller/delete.go | 4 +- internal/controller/search.go | 22 ++-- internal/index/bleve.go | 3 +- internal/index/bleve_read.go | 99 ++++++++-------- internal/index/bleve_test.go | 106 +++++++++--------- internal/index/bleve_write.go | 44 ++++++-- internal/index/document.go | 12 ++ internal/metadata/epub.go | 9 +- internal/metadata/metadata.go | 5 - .../webserver/embedded/views/document.html | 2 +- 10 files changed, 166 insertions(+), 140 deletions(-) create mode 100644 internal/index/document.go diff --git a/internal/controller/delete.go b/internal/controller/delete.go index 359868d..39f5a90 100644 --- a/internal/controller/delete.go +++ b/internal/controller/delete.go @@ -7,13 +7,13 @@ import ( "github.com/gofiber/fiber/v2" "github.com/spf13/afero" + "github.com/svera/coreander/v3/internal/index" "github.com/svera/coreander/v3/internal/jwtclaimsreader" - "github.com/svera/coreander/v3/internal/metadata" "github.com/svera/coreander/v3/internal/model" ) type IdxWriter interface { - Document(ID string) (metadata.Metadata, error) + Document(ID string) (index.Document, error) RemoveFile(file string) error } diff --git a/internal/controller/search.go b/internal/controller/search.go index a706398..96b0a9f 100644 --- a/internal/controller/search.go +++ b/internal/controller/search.go @@ -4,20 +4,12 @@ import ( "strconv" "github.com/gofiber/fiber/v2" + "github.com/svera/coreander/v3/internal/index" "github.com/svera/coreander/v3/internal/infrastructure" "github.com/svera/coreander/v3/internal/jwtclaimsreader" - "github.com/svera/coreander/v3/internal/metadata" "github.com/svera/coreander/v3/internal/model" ) -// PaginatedResult holds the result of a search request, as well as some related metadata -type PaginatedResult struct { - Page int - TotalPages int - Hits []metadata.Metadata - TotalHits int -} - type Sender interface { SendDocument(address string, libraryPath string, fileName string) error From() string @@ -25,13 +17,13 @@ type Sender interface { // IdxReader defines a set of reading operations over an index type IdxReader interface { - Search(keywords string, page, resultsPerPage int) (*PaginatedResult, error) + Search(keywords string, page, resultsPerPage int) (*index.PaginatedResult, error) Count() (uint64, error) Close() error - Document(ID string) (metadata.Metadata, error) - SameSubjects(slug string, quantity int) ([]metadata.Metadata, error) - SameAuthors(slug string, quantity int) ([]metadata.Metadata, error) - SameSeries(slug string, quantity int) ([]metadata.Metadata, error) + Document(ID string) (index.Document, error) + SameSubjects(slug string, quantity int) ([]index.Document, error) + SameAuthors(slug string, quantity int) ([]index.Document, error) + SameSeries(slug string, quantity int) ([]index.Document, error) } func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64) error { @@ -50,7 +42,7 @@ func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64) wordsPerMinute = session.WordsPerMinute } - var searchResults *PaginatedResult + var searchResults *index.PaginatedResult if keywords := c.Query("search"); keywords != "" { if searchResults, err = idx.Search(keywords, page, model.ResultsPerPage); err != nil { diff --git a/internal/index/bleve.go b/internal/index/bleve.go index f1333f1..1306ad1 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -2,6 +2,7 @@ package index import ( "log" + "path/filepath" "strings" "github.com/blevesearch/bleve/analysis/token/lowercase" @@ -23,7 +24,7 @@ type BleveIndexer struct { func NewBleve(index bleve.Index, libraryPath string, read map[string]metadata.Reader) *BleveIndexer { return &BleveIndexer{ index, - strings.TrimSuffix(libraryPath, "/"), + strings.TrimSuffix(libraryPath, string(filepath.Separator)), read, } } diff --git a/internal/index/bleve_read.go b/internal/index/bleve_read.go index 1283879..1547861 100644 --- a/internal/index/bleve_read.go +++ b/internal/index/bleve_read.go @@ -11,12 +11,19 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search/query" "github.com/gosimple/slug" - "github.com/svera/coreander/v3/internal/controller" "github.com/svera/coreander/v3/internal/metadata" ) +// PaginatedResult holds the result of a search request, as well as some related metadata +type PaginatedResult struct { + Page int + TotalPages int + Hits []Document + TotalHits int +} + // Search look for documents which match with the passed keywords. Returns a maximum books, offset by -func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*controller.PaginatedResult, error) { +func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*PaginatedResult, error) { for _, prefix := range []string{"Authors:", "Series:", "Title:", "Subjects:", "\""} { if strings.HasPrefix(strings.Trim(keywords, " "), prefix) { query := bleve.NewQueryStringQuery(keywords) @@ -89,7 +96,7 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*contr return b.runPaginatedQuery(compound, page, resultsPerPage) } -func (b *BleveIndexer) runQuery(query query.Query, results int) ([]metadata.Metadata, error) { +func (b *BleveIndexer) runQuery(query query.Query, results int) ([]Document, error) { res, err := b.runPaginatedQuery(query, 0, results) if err != nil { return nil, err @@ -97,8 +104,8 @@ func (b *BleveIndexer) runQuery(query query.Query, results int) ([]metadata.Meta return res.Hits, nil } -func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (*controller.PaginatedResult, error) { - var result controller.PaginatedResult +func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (*PaginatedResult, error) { + var result PaginatedResult if page < 1 { page = 1 } @@ -124,28 +131,30 @@ func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage return nil, err } } - result = controller.PaginatedResult{ + result = PaginatedResult{ Page: page, TotalPages: totalPages, TotalHits: int(searchResult.Total), - Hits: make([]metadata.Metadata, 0, len(searchResult.Hits)), + Hits: make([]Document, 0, len(searchResult.Hits)), } for _, val := range searchResult.Hits { - doc := metadata.Metadata{ - ID: val.ID, - BaseName: filepath.Base(val.ID), - Slug: val.Fields["Slug"].(string), - Title: val.Fields["Title"].(string), - Authors: slicer(val.Fields["Authors"]), - Description: template.HTML(val.Fields["Description"].(string)), - Year: val.Fields["Year"].(string), - Words: val.Fields["Words"].(float64), - Series: val.Fields["Series"].(string), - SeriesIndex: val.Fields["SeriesIndex"].(float64), - Pages: int(val.Fields["Pages"].(float64)), - Type: val.Fields["Type"].(string), - Subjects: slicer(val.Fields["Subjects"]), + doc := Document{ + ID: val.ID, + BaseName: filepath.Base(val.ID), + Slug: val.Fields["Slug"].(string), + Metadata: metadata.Metadata{ + Title: val.Fields["Title"].(string), + Authors: slicer(val.Fields["Authors"]), + Description: template.HTML(val.Fields["Description"].(string)), + Year: val.Fields["Year"].(string), + Words: val.Fields["Words"].(float64), + Series: val.Fields["Series"].(string), + SeriesIndex: val.Fields["SeriesIndex"].(float64), + Pages: int(val.Fields["Pages"].(float64)), + Type: val.Fields["Type"].(string), + Subjects: slicer(val.Fields["Subjects"]), + }, } result.Hits = append(result.Hits, doc) } @@ -161,8 +170,8 @@ func calculateTotalPages(total, resultsPerPage uint64) int { return int(math.Ceil(float64(total) / float64(resultsPerPage))) } -func (b *BleveIndexer) Document(slug string) (metadata.Metadata, error) { - doc := metadata.Metadata{} +func (b *BleveIndexer) Document(slug string) (Document, error) { + doc := Document{} query := bleve.NewTermQuery(slug) query.SetField("Slug") searchOptions := bleve.NewSearchRequest(query) @@ -175,20 +184,22 @@ func (b *BleveIndexer) Document(slug string) (metadata.Metadata, error) { return doc, fmt.Errorf("Document with slug %s not found", slug) } - doc = metadata.Metadata{ - ID: searchResult.Hits[0].ID, - BaseName: filepath.Base(searchResult.Hits[0].ID), - Slug: searchResult.Hits[0].Fields["Slug"].(string), - Title: searchResult.Hits[0].Fields["Title"].(string), - Authors: slicer(searchResult.Hits[0].Fields["Authors"]), - Description: template.HTML(searchResult.Hits[0].Fields["Description"].(string)), - Year: searchResult.Hits[0].Fields["Year"].(string), - Words: searchResult.Hits[0].Fields["Words"].(float64), - Series: searchResult.Hits[0].Fields["Series"].(string), - SeriesIndex: searchResult.Hits[0].Fields["SeriesIndex"].(float64), - Pages: int(searchResult.Hits[0].Fields["Pages"].(float64)), - Type: searchResult.Hits[0].Fields["Type"].(string), - Subjects: slicer(searchResult.Hits[0].Fields["Subjects"]), + doc = Document{ + ID: searchResult.Hits[0].ID, + BaseName: filepath.Base(searchResult.Hits[0].ID), + Slug: searchResult.Hits[0].Fields["Slug"].(string), + Metadata: metadata.Metadata{ + Title: searchResult.Hits[0].Fields["Title"].(string), + Authors: slicer(searchResult.Hits[0].Fields["Authors"]), + Description: template.HTML(searchResult.Hits[0].Fields["Description"].(string)), + Year: searchResult.Hits[0].Fields["Year"].(string), + Words: searchResult.Hits[0].Fields["Words"].(float64), + Series: searchResult.Hits[0].Fields["Series"].(string), + SeriesIndex: searchResult.Hits[0].Fields["SeriesIndex"].(float64), + Pages: int(searchResult.Hits[0].Fields["Pages"].(float64)), + Type: searchResult.Hits[0].Fields["Type"].(string), + Subjects: slicer(searchResult.Hits[0].Fields["Subjects"]), + }, } return doc, nil @@ -196,10 +207,10 @@ func (b *BleveIndexer) Document(slug string) (metadata.Metadata, error) { // SameSubjects returns an array of metadata of documents by other authors, different between each other, // which have similar subjects as the passed one and does not belong to the same collection -func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metadata, error) { +func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]Document, error) { doc, err := b.Document(slug) if err != nil { - return []metadata.Metadata{}, err + return []Document{}, err } subjectsCompoundQuery := bleve.NewDisjunctionQuery() @@ -222,7 +233,7 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metad } bq.AddMustNot(authorsCompoundQuery) - res := make([]metadata.Metadata, 0, quantity) + res := make([]Document, 0, quantity) for i := 0; i < quantity; i++ { doc, err := b.runQuery(bq, 1) if err != nil { @@ -245,10 +256,10 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metad // SameAuthors returns an array of metadata of documents by the same authors which // does not belong to the same collection -func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]metadata.Metadata, error) { +func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]Document, error) { doc, err := b.Document(slug) if err != nil { - return []metadata.Metadata{}, err + return []Document{}, err } authorsCompoundQuery := bleve.NewDisjunctionQuery() @@ -268,10 +279,10 @@ func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]metadata.Metada } // SameSeries returns an array of metadata of documents in the same series -func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]metadata.Metadata, error) { +func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]Document, error) { doc, err := b.Document(slug) if err != nil { - return []metadata.Metadata{}, err + return []Document{}, err } bq := bleve.NewBooleanQuery() diff --git a/internal/index/bleve_test.go b/internal/index/bleve_test.go index a0f423a..ebcd20d 100644 --- a/internal/index/bleve_test.go +++ b/internal/index/bleve_test.go @@ -6,7 +6,6 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/spf13/afero" - "github.com/svera/coreander/v3/internal/controller" "github.com/svera/coreander/v3/internal/index" "github.com/svera/coreander/v3/internal/metadata" ) @@ -54,7 +53,7 @@ type testCase struct { filename string mockedMeta metadata.Metadata search string - expectedResult controller.PaginatedResult + expectedResult index.PaginatedResult } func testCases() []testCase { @@ -63,7 +62,6 @@ func testCases() []testCase { "Look for a term without accent must return accented results", "lib/book1.epub", metadata.Metadata{ - Slug: "perez-test-a", Title: "Test A", Authors: []string{"Pérez"}, Description: "Just test metadata", @@ -71,19 +69,21 @@ func testCases() []testCase { Subjects: []string{"History", "Middle age"}, }, "perez", - controller.PaginatedResult{ + index.PaginatedResult{ Page: 1, TotalPages: 1, TotalHits: 1, - Hits: []metadata.Metadata{ + Hits: []index.Document{ { - ID: "book1.epub", - BaseName: "book1.epub", - Slug: "perez-test-a", - Title: "Test A", - Authors: []string{"Pérez"}, - Description: "Just test metadata", - Subjects: []string{"History", "Middle age"}, + ID: "book1.epub", + BaseName: "book1.epub", + Slug: "perez-test-a", + Metadata: metadata.Metadata{ + Title: "Test A", + Authors: []string{"Pérez"}, + Description: "Just test metadata", + Subjects: []string{"History", "Middle age"}, + }, }, }, }, @@ -93,26 +93,27 @@ func testCases() []testCase { "lib/book2.epub", metadata.Metadata{ Title: "Test B", - Slug: "benoit-test-b", Authors: []string{"Benoît"}, Description: "Just test metadata", Language: "fr", Subjects: []string{""}, }, "benoit", - controller.PaginatedResult{ + index.PaginatedResult{ Page: 1, TotalPages: 1, TotalHits: 1, - Hits: []metadata.Metadata{ + Hits: []index.Document{ { - ID: "book2.epub", - BaseName: "book2.epub", - Slug: "benoit-test-b", - Title: "Test B", - Authors: []string{"Benoît"}, - Description: "Just test metadata", - Subjects: []string{""}, + ID: "book2.epub", + BaseName: "book2.epub", + Slug: "benoit-test-b", + Metadata: metadata.Metadata{ + Title: "Test B", + Authors: []string{"Benoît"}, + Description: "Just test metadata", + Subjects: []string{""}, + }, }, }, }, @@ -122,26 +123,27 @@ func testCases() []testCase { "lib/book3.epub", metadata.Metadata{ Title: "Test C", - Slug: "clifford-d-simak-test-c", Authors: []string{"Clifford D. Simak"}, Description: "Just test metadata", Language: "en", Subjects: []string{""}, }, "clifford simak", - controller.PaginatedResult{ + index.PaginatedResult{ Page: 1, TotalPages: 1, TotalHits: 1, - Hits: []metadata.Metadata{ + Hits: []index.Document{ { - ID: "book3.epub", - BaseName: "book3.epub", - Slug: "clifford-d-simak-test-c", - Title: "Test C", - Authors: []string{"Clifford D. Simak"}, - Description: "Just test metadata", - Subjects: []string{""}, + ID: "book3.epub", + BaseName: "book3.epub", + Slug: "clifford-d-simak-test-c", + Metadata: metadata.Metadata{ + Title: "Test C", + Authors: []string{"Clifford D. Simak"}, + Description: "Just test metadata", + Subjects: []string{""}, + }, }, }, }, @@ -151,26 +153,26 @@ func testCases() []testCase { "lib/book4.epub", metadata.Metadata{ Title: "Test D", - Slug: "james-ellroy-test-d", Authors: []string{"James Ellroy"}, Description: "Just test metadata", Language: "en", Subjects: []string{""}, }, "james ellroy", - controller.PaginatedResult{ + index.PaginatedResult{ Page: 1, TotalPages: 1, TotalHits: 1, - Hits: []metadata.Metadata{ + Hits: []index.Document{ { - ID: "book4.epub", - BaseName: "book4.epub", - Slug: "james-ellroy-test-d", - Title: "Test D", - Authors: []string{"James Ellroy"}, - Description: "Just test metadata", - Subjects: []string{""}, + ID: "book4.epub", + BaseName: "book4.epub", + Slug: "james-ellroy-test-d", + Metadata: metadata.Metadata{Title: "Test D", + Authors: []string{"James Ellroy"}, + Description: "Just test metadata", + Subjects: []string{""}, + }, }, }, }, @@ -180,26 +182,26 @@ func testCases() []testCase { "lib/book5.epub", metadata.Metadata{ Title: "Test E", - Slug: "james-ellroy-test-e", Authors: []string{"James Ellroy"}, Description: "Just test metadata", Language: "en", Subjects: []string{""}, }, " james ellroy ", - controller.PaginatedResult{ + index.PaginatedResult{ Page: 1, TotalPages: 1, TotalHits: 1, - Hits: []metadata.Metadata{ + Hits: []index.Document{ { - ID: "book5.epub", - BaseName: "book5.epub", - Slug: "james-ellroy-test-e", - Title: "Test E", - Authors: []string{"James Ellroy"}, - Description: "Just test metadata", - Subjects: []string{""}, + ID: "book5.epub", + BaseName: "book5.epub", + Slug: "james-ellroy-test-e", + Metadata: metadata.Metadata{Title: "Test E", + Authors: []string{"James Ellroy"}, + Description: "Just test metadata", + Subjects: []string{""}, + }, }, }, }, diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index b3fab8d..bca8a3c 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -9,7 +9,6 @@ import ( "github.com/gosimple/slug" "github.com/spf13/afero" - "github.com/svera/coreander/v3/internal/metadata" ) // AddFile adds a file to the index @@ -23,11 +22,21 @@ func (b *BleveIndexer) AddFile(file string) error { return fmt.Errorf("error extracting metadata from file %s: %s", file, err) } - docSlug := makeSlug(meta) - meta = b.setID(meta, file) - meta.Slug = b.checkSlug(meta.ID, docSlug, nil) + document := Document{ + Metadata: meta, + } + + docSlug := makeSlug(document) + document = b.setID(document, file) + document.Slug = b.checkSlug(document.ID, docSlug, nil) + document.SeriesEq = strings.ReplaceAll(slug.Make(document.Series), "-", "") + document.AuthorsEq = make([]string, len(document.Authors)) + copy(document.AuthorsEq, meta.Authors) + for i := range document.AuthorsEq { + document.AuthorsEq[i] = strings.ReplaceAll(slug.Make(document.AuthorsEq[i]), "-", "") + } - err = b.idx.Index(meta.ID, meta) + err = b.idx.Index(document.ID, document) if err != nil { return fmt.Errorf("error indexing file %s: %s", file, err) } @@ -37,7 +46,7 @@ func (b *BleveIndexer) AddFile(file string) error { // RemoveFile removes a file from the index func (b *BleveIndexer) RemoveFile(file string) error { file = strings.Replace(file, b.libraryPath, "", 1) - file = strings.TrimPrefix(file, "/") + file = strings.TrimPrefix(file, string(filepath.Separator)) if err := b.idx.Delete(file); err != nil { return err } @@ -59,12 +68,23 @@ func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error { return nil } - docSlug := makeSlug(meta) - meta = b.setID(meta, fullPath) - meta.Slug = b.checkSlug(meta.ID, docSlug, batchSlugs) + document := Document{ + Metadata: meta, + } + + docSlug := makeSlug(document) + document = b.setID(document, fullPath) + document.Slug = b.checkSlug(document.ID, docSlug, batchSlugs) + document.SeriesEq = strings.ReplaceAll(slug.Make(document.Series), "-", "") + document.AuthorsEq = make([]string, len(document.Authors)) + copy(document.AuthorsEq, meta.Authors) + for i := range document.AuthorsEq { + document.AuthorsEq[i] = strings.ReplaceAll(slug.Make(document.AuthorsEq[i]), "-", "") + } + batchSlugs[docSlug] = struct{}{} - err = batch.Index(meta.ID, meta) + err = batch.Index(document.ID, document) if err != nil { log.Printf("Error indexing file %s: %s\n", fullPath, err) return nil @@ -103,14 +123,14 @@ func (b *BleveIndexer) checkSlug(ID, docSlug string, batchSlugs map[string]struc } } -func (b *BleveIndexer) setID(meta metadata.Metadata, file string) metadata.Metadata { +func (b *BleveIndexer) setID(meta Document, file string) Document { meta.ID = strings.ReplaceAll(file, b.libraryPath, "") meta.ID = strings.TrimPrefix(meta.ID, "/") return meta } -func makeSlug(meta metadata.Metadata) string { +func makeSlug(meta Document) string { docSlug := meta.Title if len(meta.Authors) > 0 { docSlug = strings.Join(meta.Authors, ", ") + "-" + docSlug diff --git a/internal/index/document.go b/internal/index/document.go new file mode 100644 index 0000000..43cc590 --- /dev/null +++ b/internal/index/document.go @@ -0,0 +1,12 @@ +package index + +import "github.com/svera/coreander/v3/internal/metadata" + +type Document struct { + metadata.Metadata + ID string + BaseName string + Slug string + AuthorsEq []string + SeriesEq string +} diff --git a/internal/metadata/epub.go b/internal/metadata/epub.go index ac351d0..bbd0b24 100644 --- a/internal/metadata/epub.go +++ b/internal/metadata/epub.go @@ -14,7 +14,6 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/disintegration/imaging" "github.com/gofiber/fiber/v2" - "github.com/gosimple/slug" "github.com/microcosm-cc/bluemonday" "github.com/pirmd/epub" ) @@ -31,21 +30,17 @@ func (e EpubReader) Metadata(file string) (Metadata, error) { if len(opf.Metadata.Title) > 0 && len(opf.Metadata.Title[0].Value) > 0 { title = opf.Metadata.Title[0].Value } - var authors, authorsEq []string + var authors []string if len(opf.Metadata.Creator) > 0 { for _, creator := range opf.Metadata.Creator { if creator.Role == "aut" || creator.Role == "" { // Some epub files mistakenly put all authors in a single field instead of using a field for each one. // We want to identify those cases looking for specific separators and then indexing each author properly. names := strings.Split(creator.Value, "&") - namesEq := make([]string, len(names)) - copy(namesEq, names) for i := range names { names[i] = strings.TrimSpace(names[i]) - namesEq[i] = strings.ReplaceAll(slug.Make(namesEq[i]), "-", "") } authors = append(authors, names...) - authorsEq = append(authorsEq, namesEq...) } } } @@ -120,13 +115,11 @@ func (e EpubReader) Metadata(file string) (Metadata, error) { bk = Metadata{ Title: title, Authors: authors, - AuthorsEq: authorsEq, Description: template.HTML(description), Language: language, Year: year, Cover: cover, Series: series, - SeriesEq: strings.ReplaceAll(slug.Make(series), "-", ""), SeriesIndex: seriesIndex, Type: "EPUB", Subjects: subjects, diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go index b93088b..a290234 100644 --- a/internal/metadata/metadata.go +++ b/internal/metadata/metadata.go @@ -7,19 +7,14 @@ import ( ) type Metadata struct { - ID string - BaseName string - Slug string Title string Authors []string - AuthorsEq []string Description template.HTML Language string Year string Words float64 Cover string Series string - SeriesEq string SeriesIndex float64 Pages int Type string diff --git a/internal/webserver/embedded/views/document.html b/internal/webserver/embedded/views/document.html index 2d4c290..a446af9 100644 --- a/internal/webserver/embedded/views/document.html +++ b/internal/webserver/embedded/views/document.html @@ -72,7 +72,7 @@

{{range $i, $author := .Document.Authors}} {{$authorTitle := t $lang "Search for more titles by %s" $author}} - {{$author}}{{if notLast $document.Authors $i}}, {{end}} {{end}}

From 7e9740ddcc9ef9fe29471449c80a94e07e715e20 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 25 Sep 2023 21:14:20 +0200 Subject: [PATCH 3/8] Added see more to same subjects, improved markup of document detail --- internal/controller/document.go | 1 + internal/index/bleve_read.go | 2 +- internal/index/bleve_write.go | 26 ++++++-- internal/index/document.go | 17 ++++-- .../webserver/embedded/views/document.html | 61 +++++-------------- .../embedded/views/partials/related.html | 4 +- 6 files changed, 50 insertions(+), 61 deletions(-) diff --git a/internal/controller/document.go b/internal/controller/document.go index 30be560..4d99a4a 100644 --- a/internal/controller/document.go +++ b/internal/controller/document.go @@ -61,6 +61,7 @@ func Document(c *fiber.Ctx, libraryPath string, sender Sender, idx IdxReader, wo "Title": title, "Document": document, "Authors": strings.Join(document.Authors, ","), + "Subjects": strings.Join(document.Subjects, ","), "EmailSendingConfigured": emailSendingConfigured, "EmailFrom": sender.From(), "Session": session, diff --git a/internal/index/bleve_read.go b/internal/index/bleve_read.go index 1547861..c37d44c 100644 --- a/internal/index/bleve_read.go +++ b/internal/index/bleve_read.go @@ -32,7 +32,7 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*Pagin } } - for _, prefix := range []string{"AuthorsEq:", "SeriesEq:"} { + for _, prefix := range []string{"AuthorsEq:", "SeriesEq:", "SubjectsEq:"} { unescaped, err := url.QueryUnescape(strings.TrimSpace(keywords)) if err != nil { break diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index bca8a3c..3235e12 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -22,8 +22,10 @@ func (b *BleveIndexer) AddFile(file string) error { return fmt.Errorf("error extracting metadata from file %s: %s", file, err) } - document := Document{ - Metadata: meta, + document := DocumentWrite{ + Document: Document{ + Metadata: meta, + }, } docSlug := makeSlug(document) @@ -35,6 +37,11 @@ func (b *BleveIndexer) AddFile(file string) error { for i := range document.AuthorsEq { document.AuthorsEq[i] = strings.ReplaceAll(slug.Make(document.AuthorsEq[i]), "-", "") } + document.SubjectsEq = make([]string, len(document.Subjects)) + copy(document.SubjectsEq, meta.Subjects) + for i := range document.SubjectsEq { + document.SubjectsEq[i] = strings.ReplaceAll(slug.Make(document.SubjectsEq[i]), "-", "") + } err = b.idx.Index(document.ID, document) if err != nil { @@ -68,8 +75,10 @@ func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error { return nil } - document := Document{ - Metadata: meta, + document := DocumentWrite{ + Document: Document{ + Metadata: meta, + }, } docSlug := makeSlug(document) @@ -81,6 +90,11 @@ func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error { for i := range document.AuthorsEq { document.AuthorsEq[i] = strings.ReplaceAll(slug.Make(document.AuthorsEq[i]), "-", "") } + document.SubjectsEq = make([]string, len(document.Subjects)) + copy(document.SubjectsEq, meta.Subjects) + for i := range document.SubjectsEq { + document.SubjectsEq[i] = strings.ReplaceAll(slug.Make(document.SubjectsEq[i]), "-", "") + } batchSlugs[docSlug] = struct{}{} @@ -123,14 +137,14 @@ func (b *BleveIndexer) checkSlug(ID, docSlug string, batchSlugs map[string]struc } } -func (b *BleveIndexer) setID(meta Document, file string) Document { +func (b *BleveIndexer) setID(meta DocumentWrite, file string) DocumentWrite { meta.ID = strings.ReplaceAll(file, b.libraryPath, "") meta.ID = strings.TrimPrefix(meta.ID, "/") return meta } -func makeSlug(meta Document) string { +func makeSlug(meta DocumentWrite) string { docSlug := meta.Title if len(meta.Authors) > 0 { docSlug = strings.Join(meta.Authors, ", ") + "-" + docSlug diff --git a/internal/index/document.go b/internal/index/document.go index 43cc590..6ede1d5 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -4,9 +4,16 @@ import "github.com/svera/coreander/v3/internal/metadata" type Document struct { metadata.Metadata - ID string - BaseName string - Slug string - AuthorsEq []string - SeriesEq string + ID string + BaseName string + Slug string +} + +// DocumentWrite is an extension to Document that is used only when writing to the index, +// as some of its fields are only used to perform searches and not returned +type DocumentWrite struct { + Document + AuthorsEq []string + SeriesEq string + SubjectsEq []string } diff --git a/internal/webserver/embedded/views/document.html b/internal/webserver/embedded/views/document.html index a446af9..2ce8a18 100644 --- a/internal/webserver/embedded/views/document.html +++ b/internal/webserver/embedded/views/document.html @@ -10,7 +10,7 @@
-
+
{{t $lang "\"%s\" cover" .Document.Title}} @@ -54,7 +54,7 @@ {{end}}
-
+
{{ if ne .Document.Series "" }} {{$seriesTitle := t $lang "Search for more titles belonging to %s" @@ -72,8 +72,7 @@

{{range $i, $author := .Document.Authors}} {{$authorTitle := t $lang "Search for more titles by %s" $author}} - {{$author}}{{if notLast $document.Authors $i}}, {{end}} + {{$author}}{{if notLast $document.Authors $i}}, {{end}} {{end}}

{{else}} @@ -93,7 +92,7 @@

{{t $lang "Unknown author"}}

{{range $i, $subject := .Document.Subjects}} {{$subjectTitle := t $lang "Search for more titles in %s" $subject}} - {{$subject}} {{end}}
@@ -109,10 +108,10 @@

{{t $lang "Unknown author"}}

{{ $length := len .SameSeries }} {{ if gt $length 0 }}
-
+

{{t $lang "Other documents in collection \"%s\"" .Document.Series}}

-
+
{{t $lang "See all" }}
@@ -123,7 +122,7 @@

{{t $lang "Other documents in collection \"%s\"" .Document.Series}}

{{ $length := len .SameAuthors }} {{ if gt $length 0 }}
-
+

{{t $lang "Other documents by"}} {{range $i, $author := $document.Authors}} @@ -131,7 +130,7 @@

{{end}}

-
+
{{t $lang "See all" }}
@@ -142,9 +141,14 @@

{{ $length := len .SameSubjects }} {{ if gt $length 0 }}
-
+

{{t $lang "Other documents with similar subjects"}}

+
+ + {{t $lang "See all" }} +
+ {{template "partials/related" dict "Lang" $lang "Related" .SameSubjects}}
{{end}} @@ -153,18 +157,7 @@

{{t $lang "Other documents with similar subjects"}}

-{{template "partials/delete-modal" .}} -