Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add see more link to related documents' sections #47

Merged
merged 9 commits into from
Sep 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions internal/controller/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ import (

"github.com/gofiber/fiber/v2"
"github.com/spf13/afero"
"github.com/svera/coreander/v3/internal/index"
"github.com/svera/coreander/v3/internal/jwtclaimsreader"
"github.com/svera/coreander/v3/internal/metadata"
"github.com/svera/coreander/v3/internal/model"
)

type IdxWriter interface {
Document(ID string) (metadata.Metadata, error)
Document(ID string) (index.Document, error)
RemoveFile(file string) error
}

Expand Down
23 changes: 8 additions & 15 deletions internal/controller/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,26 @@ import (
"strconv"

"github.com/gofiber/fiber/v2"
"github.com/svera/coreander/v3/internal/index"
"github.com/svera/coreander/v3/internal/infrastructure"
"github.com/svera/coreander/v3/internal/jwtclaimsreader"
"github.com/svera/coreander/v3/internal/metadata"
"github.com/svera/coreander/v3/internal/model"
)

// PaginatedResult holds the result of a search request, as well as some related metadata
type PaginatedResult struct {
Page int
TotalPages int
Hits []metadata.Metadata
TotalHits int
}

type Sender interface {
SendDocument(address string, libraryPath string, fileName string) error
From() string
}

// IdxReader defines a set of reading operations over an index
type IdxReader interface {
Search(keywords string, page, resultsPerPage int) (*PaginatedResult, error)
Search(keywords string, page, resultsPerPage int) (*index.PaginatedResult, error)
Count() (uint64, error)
Close() error
Document(ID string) (metadata.Metadata, error)
SameSubjects(slug string, quantity int) ([]metadata.Metadata, error)
SameAuthors(slug string, quantity int) ([]metadata.Metadata, error)
SameSeries(slug string, quantity int) ([]metadata.Metadata, error)
Document(ID string) (index.Document, error)
SameSubjects(slug string, quantity int) ([]index.Document, error)
SameAuthors(slug string, quantity int) ([]index.Document, error)
SameSeries(slug string, quantity int) ([]index.Document, error)
}

func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64) error {
Expand All @@ -50,7 +42,7 @@ func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64)
wordsPerMinute = session.WordsPerMinute
}

var searchResults *PaginatedResult
var searchResults *index.PaginatedResult

if keywords := c.Query("search"); keywords != "" {
if searchResults, err = idx.Search(keywords, page, model.ResultsPerPage); err != nil {
Expand All @@ -70,6 +62,7 @@ func Search(c *fiber.Ctx, idx IdxReader, sender Sender, wordsPerMinute float64)
"WordsPerMinute": wordsPerMinute,
}, "layout")
}

count, err := idx.Count()
if err != nil {
return fiber.ErrInternalServerError
Expand Down
9 changes: 8 additions & 1 deletion internal/index/bleve.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package index

import (
"log"
"path/filepath"
"strings"

"github.com/blevesearch/bleve/analysis/token/lowercase"
Expand All @@ -23,7 +24,7 @@ type BleveIndexer struct {
func NewBleve(index bleve.Index, libraryPath string, read map[string]metadata.Reader) *BleveIndexer {
return &BleveIndexer{
index,
strings.TrimSuffix(libraryPath, "/"),
strings.TrimSuffix(libraryPath, string(filepath.Separator)),
read,
}
}
Expand Down Expand Up @@ -54,6 +55,12 @@ func Mapping() *mapping.IndexMappingImpl {
indexMapping.DefaultMapping.AddFieldMappingsAt("Year", yearFieldMapping)
slugFieldMapping := bleve.NewKeywordFieldMapping()
indexMapping.DefaultMapping.AddFieldMappingsAt("Slug", slugFieldMapping)
seriesEqFieldMapping := bleve.NewKeywordFieldMapping()
indexMapping.DefaultMapping.AddFieldMappingsAt("SeriesEq", seriesEqFieldMapping)
authorsEqFieldMapping := bleve.NewKeywordFieldMapping()
indexMapping.DefaultMapping.AddFieldMappingsAt("AuthorsEq", authorsEqFieldMapping)
subjectsEqFieldMapping := bleve.NewKeywordFieldMapping()
indexMapping.DefaultMapping.AddFieldMappingsAt("SubjectsEq", subjectsEqFieldMapping)

return indexMapping
}
Expand Down
112 changes: 67 additions & 45 deletions internal/index/bleve_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ import (
"fmt"
"html/template"
"math"
"path/filepath"
"net/url"
"strings"

"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/svera/coreander/v3/internal/controller"
"github.com/gosimple/slug"
"github.com/svera/coreander/v3/internal/metadata"
)

// Search look for documents which match with the passed keywords. Returns a maximum <resultsPerPage> books, offset by <page>
func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*controller.PaginatedResult, error) {
func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*PaginatedResult, error) {
for _, prefix := range []string{"Authors:", "Series:", "Title:", "Subjects:", "\""} {
if strings.HasPrefix(strings.Trim(keywords, " "), prefix) {
query := bleve.NewQueryStringQuery(keywords)
Expand All @@ -23,6 +23,26 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*contr
}
}

for _, prefix := range []string{"AuthorsEq:", "SeriesEq:", "SubjectsEq:"} {
unescaped, err := url.QueryUnescape(strings.TrimSpace(keywords))
if err != nil {
break
}
if !strings.HasPrefix(unescaped, prefix) {
continue
}
unescaped = strings.TrimPrefix(unescaped, prefix)
terms := strings.Split(unescaped, ",")
qb := bleve.NewDisjunctionQuery()
for _, term := range terms {
term = strings.ReplaceAll(slug.Make(term), "-", "")
qs := bleve.NewTermQuery(term)
qs.SetField(strings.TrimSuffix(prefix, ":"))
qb.AddQuery(qs)
}
return b.runPaginatedQuery(qb, page, resultsPerPage)
}

splitted := strings.Split(strings.TrimSpace(keywords), " ")

var (
Expand Down Expand Up @@ -68,16 +88,16 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*contr
return b.runPaginatedQuery(compound, page, resultsPerPage)
}

func (b *BleveIndexer) runQuery(query query.Query, results int) ([]metadata.Metadata, error) {
func (b *BleveIndexer) runQuery(query query.Query, results int) ([]Document, error) {
res, err := b.runPaginatedQuery(query, 0, results)
if err != nil {
return nil, err
}
return res.Hits, nil
}

func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (*controller.PaginatedResult, error) {
var result controller.PaginatedResult
func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (*PaginatedResult, error) {
var result PaginatedResult
if page < 1 {
page = 1
}
Expand All @@ -103,28 +123,29 @@ func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage
return nil, err
}
}
result = controller.PaginatedResult{
result = PaginatedResult{
Page: page,
TotalPages: totalPages,
TotalHits: int(searchResult.Total),
Hits: make([]metadata.Metadata, 0, len(searchResult.Hits)),
Hits: make([]Document, 0, len(searchResult.Hits)),
}

for _, val := range searchResult.Hits {
doc := metadata.Metadata{
ID: val.ID,
BaseName: filepath.Base(val.ID),
Slug: val.Fields["Slug"].(string),
Title: val.Fields["Title"].(string),
Authors: slicer(val.Fields["Authors"]),
Description: template.HTML(val.Fields["Description"].(string)),
Year: val.Fields["Year"].(string),
Words: val.Fields["Words"].(float64),
Series: val.Fields["Series"].(string),
SeriesIndex: val.Fields["SeriesIndex"].(float64),
Pages: int(val.Fields["Pages"].(float64)),
Type: val.Fields["Type"].(string),
Subjects: slicer(val.Fields["Subjects"]),
doc := Document{
ID: val.ID,
Slug: val.Fields["Slug"].(string),
Metadata: metadata.Metadata{
Title: val.Fields["Title"].(string),
Authors: slicer(val.Fields["Authors"]),
Description: template.HTML(val.Fields["Description"].(string)),
Year: val.Fields["Year"].(string),
Words: val.Fields["Words"].(float64),
Series: val.Fields["Series"].(string),
SeriesIndex: val.Fields["SeriesIndex"].(float64),
Pages: int(val.Fields["Pages"].(float64)),
Type: val.Fields["Type"].(string),
Subjects: slicer(val.Fields["Subjects"]),
},
}
result.Hits = append(result.Hits, doc)
}
Expand All @@ -140,8 +161,8 @@ func calculateTotalPages(total, resultsPerPage uint64) int {
return int(math.Ceil(float64(total) / float64(resultsPerPage)))
}

func (b *BleveIndexer) Document(slug string) (metadata.Metadata, error) {
doc := metadata.Metadata{}
func (b *BleveIndexer) Document(slug string) (Document, error) {
doc := Document{}
query := bleve.NewTermQuery(slug)
query.SetField("Slug")
searchOptions := bleve.NewSearchRequest(query)
Expand All @@ -154,31 +175,32 @@ func (b *BleveIndexer) Document(slug string) (metadata.Metadata, error) {
return doc, fmt.Errorf("Document with slug %s not found", slug)
}

doc = metadata.Metadata{
ID: searchResult.Hits[0].ID,
BaseName: filepath.Base(searchResult.Hits[0].ID),
Slug: searchResult.Hits[0].Fields["Slug"].(string),
Title: searchResult.Hits[0].Fields["Title"].(string),
Authors: slicer(searchResult.Hits[0].Fields["Authors"]),
Description: template.HTML(searchResult.Hits[0].Fields["Description"].(string)),
Year: searchResult.Hits[0].Fields["Year"].(string),
Words: searchResult.Hits[0].Fields["Words"].(float64),
Series: searchResult.Hits[0].Fields["Series"].(string),
SeriesIndex: searchResult.Hits[0].Fields["SeriesIndex"].(float64),
Pages: int(searchResult.Hits[0].Fields["Pages"].(float64)),
Type: searchResult.Hits[0].Fields["Type"].(string),
Subjects: slicer(searchResult.Hits[0].Fields["Subjects"]),
doc = Document{
ID: searchResult.Hits[0].ID,
Slug: searchResult.Hits[0].Fields["Slug"].(string),
Metadata: metadata.Metadata{
Title: searchResult.Hits[0].Fields["Title"].(string),
Authors: slicer(searchResult.Hits[0].Fields["Authors"]),
Description: template.HTML(searchResult.Hits[0].Fields["Description"].(string)),
Year: searchResult.Hits[0].Fields["Year"].(string),
Words: searchResult.Hits[0].Fields["Words"].(float64),
Series: searchResult.Hits[0].Fields["Series"].(string),
SeriesIndex: searchResult.Hits[0].Fields["SeriesIndex"].(float64),
Pages: int(searchResult.Hits[0].Fields["Pages"].(float64)),
Type: searchResult.Hits[0].Fields["Type"].(string),
Subjects: slicer(searchResult.Hits[0].Fields["Subjects"]),
},
}

return doc, nil
}

// SameSubjects returns an array of metadata of documents by other authors, different between each other,
// which have similar subjects as the passed one and does not belong to the same collection
func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metadata, error) {
func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []metadata.Metadata{}, err
return []Document{}, err
}

subjectsCompoundQuery := bleve.NewDisjunctionQuery()
Expand All @@ -201,7 +223,7 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metad
}
bq.AddMustNot(authorsCompoundQuery)

res := make([]metadata.Metadata, 0, quantity)
res := make([]Document, 0, quantity)
for i := 0; i < quantity; i++ {
doc, err := b.runQuery(bq, 1)
if err != nil {
Expand All @@ -224,10 +246,10 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]metadata.Metad

// SameAuthors returns an array of metadata of documents by the same authors which
// does not belong to the same collection
func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]metadata.Metadata, error) {
func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []metadata.Metadata{}, err
return []Document{}, err
}

authorsCompoundQuery := bleve.NewDisjunctionQuery()
Expand All @@ -247,10 +269,10 @@ func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]metadata.Metada
}

// SameSeries returns an array of metadata of documents in the same series
func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]metadata.Metadata, error) {
func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []metadata.Metadata{}, err
return []Document{}, err
}

bq := bleve.NewBooleanQuery()
Expand Down
Loading