Skip to content

Commit

Permalink
V3.8.6
Browse files Browse the repository at this point in the history
  • Loading branch information
@svera
svera authored Nov 18, 2023
1 parent 52bbb3c commit b4d85e6
Show file tree
Hide file tree
Showing 50 changed files with 363 additions and 307 deletions.
87 changes: 37 additions & 50 deletions internal/index/bleve_read.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ import (
"github.com/blevesearch/bleve/v2/search/query"
"github.com/gosimple/slug"
"github.com/svera/coreander/v3/internal/metadata"
"github.com/svera/coreander/v3/internal/search"
"github.com/svera/coreander/v3/internal/result"
)

// Search look for documents which match with the passed keywords. Returns a maximum <resultsPerPage> documents, offset by <page>
func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*search.PaginatedResult, error) {
func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (result.Paginated[[]Document], error) {
for _, prefix := range []string{"Authors:", "Series:", "Title:", "Subjects:", "\""} {
if strings.HasPrefix(strings.Trim(keywords, " "), prefix) {
query := bleve.NewQueryStringQuery(keywords)
Expand Down Expand Up @@ -88,16 +88,17 @@ func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (*searc
return b.runPaginatedQuery(compound, page, resultsPerPage)
}

func (b *BleveIndexer) runQuery(query query.Query, results int) ([]search.Document, error) {
func (b *BleveIndexer) runQuery(query query.Query, results int) ([]Document, error) {
res, err := b.runPaginatedQuery(query, 0, results)
if err != nil {
return nil, err
}
return res.Hits, nil
return res.Hits(), nil
}

func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (*search.PaginatedResult, error) {
var result search.PaginatedResult
func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage int) (result.Paginated[[]Document], error) {
var res result.Paginated[[]Document]

if page < 1 {
page = 1
}
Expand All @@ -107,31 +108,16 @@ func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage
searchOptions.Fields = []string{"ID", "Slug", "Title", "Authors", "Description", "Year", "Words", "Series", "SeriesIndex", "Pages", "Type", "Subjects"}
searchResult, err := b.idx.Search(searchOptions)
if err != nil {
return nil, err
return result.Paginated[[]Document]{}, err
}
if searchResult.Total == 0 {
return &result, nil
}
totalPages := search.CalculateTotalPages(searchResult.Total, uint64(resultsPerPage))
if totalPages < page {
page = totalPages
if page == 0 {
page = 1
}
searchResult, err = b.idx.Search(searchOptions)
if err != nil {
return nil, err
}
}
result = search.PaginatedResult{
Page: page,
TotalPages: totalPages,
TotalHits: int(searchResult.Total),
Hits: make([]search.Document, 0, len(searchResult.Hits)),
return res, nil
}

docs := make([]Document, 0, len(searchResult.Hits))

for _, val := range searchResult.Hits {
doc := search.Document{
doc := Document{
ID: val.ID,
Slug: val.Fields["Slug"].(string),
Metadata: metadata.Metadata{
Expand All @@ -147,31 +133,36 @@ func (b *BleveIndexer) runPaginatedQuery(query query.Query, page, resultsPerPage
Subjects: slicer(val.Fields["Subjects"]),
},
}
result.Hits = append(result.Hits, doc)
docs = append(docs, doc)
}
return &result, nil

return result.NewPaginated[[]Document](
resultsPerPage,
page,
int(searchResult.Total),
docs,
), nil
}

// Count returns the number of indexed documents
func (b *BleveIndexer) Count() (uint64, error) {
return b.idx.DocCount()
}

func (b *BleveIndexer) Document(slug string) (search.Document, error) {
doc := search.Document{}
func (b *BleveIndexer) Document(slug string) (Document, error) {
query := bleve.NewTermQuery(slug)
query.SetField("Slug")
searchOptions := bleve.NewSearchRequest(query)
searchOptions.Fields = []string{"ID", "Slug", "Title", "Authors", "Description", "Year", "Words", "Series", "SeriesIndex", "Pages", "Type", "Subjects"}
searchResult, err := b.idx.Search(searchOptions)
if err != nil {
return doc, err
return Document{}, err
}
if searchResult.Total == 0 {
return doc, fmt.Errorf("Document with slug %s not found", slug)
return Document{}, fmt.Errorf("Document with slug %s not found", slug)
}

doc = search.Document{
return Document{
ID: searchResult.Hits[0].ID,
Slug: searchResult.Hits[0].Fields["Slug"].(string),
Metadata: metadata.Metadata{
Expand All @@ -186,13 +177,11 @@ func (b *BleveIndexer) Document(slug string) (search.Document, error) {
Type: searchResult.Hits[0].Fields["Type"].(string),
Subjects: slicer(searchResult.Hits[0].Fields["Subjects"]),
},
}

return doc, nil
}, nil
}

func (b *BleveIndexer) Documents(IDs []string) ([]search.Document, error) {
docs := make([]search.Document, 0, len(IDs))
func (b *BleveIndexer) Documents(IDs []string) (map[string]Document, error) {
docs := make(map[string]Document, len(IDs))
query := bleve.NewDocIDQuery(IDs)
searchOptions := bleve.NewSearchRequest(query)
searchOptions.Fields = []string{"ID", "Slug", "Title", "Authors", "Description", "Year", "Words", "Series", "SeriesIndex", "Pages", "Type", "Subjects"}
Expand All @@ -202,9 +191,8 @@ func (b *BleveIndexer) Documents(IDs []string) ([]search.Document, error) {
}

for _, hit := range searchResult.Hits {
docs = append(
docs,
search.Document{
docs[hit.ID] =
Document{
ID: hit.ID,
Slug: hit.Fields["Slug"].(string),
Metadata: metadata.Metadata{
Expand All @@ -219,19 +207,18 @@ func (b *BleveIndexer) Documents(IDs []string) ([]search.Document, error) {
Type: hit.Fields["Type"].(string),
Subjects: slicer(hit.Fields["Subjects"]),
},
},
)
}
}

return docs, nil
}

// SameSubjects returns an array of metadata of documents by other authors, different between each other,
// which have similar subjects as the passed one and does not belong to the same collection
func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]search.Document, error) {
func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []search.Document{}, err
return []Document{}, err
}

subjectsCompoundQuery := bleve.NewDisjunctionQuery()
Expand All @@ -254,7 +241,7 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]search.Documen
}
bq.AddMustNot(authorsCompoundQuery)

res := make([]search.Document, 0, quantity)
res := make([]Document, 0, quantity)
for i := 0; i < quantity; i++ {
doc, err := b.runQuery(bq, 1)
if err != nil {
Expand All @@ -277,10 +264,10 @@ func (b *BleveIndexer) SameSubjects(slug string, quantity int) ([]search.Documen

// SameAuthors returns an array of metadata of documents by the same authors which
// does not belong to the same collection
func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]search.Document, error) {
func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []search.Document{}, err
return []Document{}, err
}

authorsCompoundQuery := bleve.NewDisjunctionQuery()
Expand All @@ -300,10 +287,10 @@ func (b *BleveIndexer) SameAuthors(slug string, quantity int) ([]search.Document
}

// SameSeries returns an array of metadata of documents in the same series
func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]search.Document, error) {
func (b *BleveIndexer) SameSeries(slug string, quantity int) ([]Document, error) {
doc, err := b.Document(slug)
if err != nil {
return []search.Document{}, err
return []Document{}, err
}

bq := bleve.NewBooleanQuery()
Expand Down
67 changes: 34 additions & 33 deletions internal/index/bleve_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import (
"github.com/spf13/afero"
"github.com/svera/coreander/v3/internal/index"
"github.com/svera/coreander/v3/internal/metadata"
"github.com/svera/coreander/v3/internal/search"
"github.com/svera/coreander/v3/internal/result"
"github.com/svera/coreander/v3/internal/webserver/model"
)

func TestIndexAndSearch(t *testing.T) {
Expand Down Expand Up @@ -42,7 +43,7 @@ func TestIndexAndSearch(t *testing.T) {
if err != nil {
t.Errorf("Error searching: %s", err.Error())
}
if !reflect.DeepEqual(*res, tcase.expectedResult) {
if !reflect.DeepEqual(res, tcase.expectedResult) {
t.Errorf("Wrong result returned, expected %#v, got %#v", tcase.expectedResult, res)
}
})
Expand All @@ -54,7 +55,7 @@ type testCase struct {
filename string
mockedMeta metadata.Metadata
search string
expectedResult search.PaginatedResult
expectedResult result.Paginated[[]index.Document]
}

func testCases() []testCase {
Expand All @@ -70,11 +71,11 @@ func testCases() []testCase {
Subjects: []string{"History", "Middle age"},
},
"perez",
search.PaginatedResult{
Page: 1,
TotalPages: 1,
TotalHits: 1,
Hits: []search.Document{
result.NewPaginated[[]index.Document](
model.ResultsPerPage,
1,
1,
[]index.Document{
{
ID: "book1.epub",
Slug: "perez-test-a",
Expand All @@ -86,7 +87,7 @@ func testCases() []testCase {
},
},
},
},
),
},
{
"Look for a term without circumflex accent must return circumflexed results",
Expand All @@ -99,11 +100,11 @@ func testCases() []testCase {
Subjects: []string{""},
},
"benoit",
search.PaginatedResult{
Page: 1,
TotalPages: 1,
TotalHits: 1,
Hits: []search.Document{
result.NewPaginated[[]index.Document](
model.ResultsPerPage,
1,
1,
[]index.Document{
{
ID: "book2.epub",
Slug: "benoit-test-b",
Expand All @@ -115,7 +116,7 @@ func testCases() []testCase {
},
},
},
},
),
},
{
"Look for several, not exact terms must return a result with all those terms, even if there is something in between",
Expand All @@ -128,11 +129,11 @@ func testCases() []testCase {
Subjects: []string{""},
},
"clifford simak",
search.PaginatedResult{
Page: 1,
TotalPages: 1,
TotalHits: 1,
Hits: []search.Document{
result.NewPaginated[[]index.Document](
model.ResultsPerPage,
1,
1,
[]index.Document{
{
ID: "book3.epub",
Slug: "clifford-d-simak-test-c",
Expand All @@ -144,7 +145,7 @@ func testCases() []testCase {
},
},
},
},
),
},
{
"Look for several, not exact terms must return a result with all those terms, even if there is something in between",
Expand All @@ -157,11 +158,11 @@ func testCases() []testCase {
Subjects: []string{""},
},
"james ellroy",
search.PaginatedResult{
Page: 1,
TotalPages: 1,
TotalHits: 1,
Hits: []search.Document{
result.NewPaginated[[]index.Document](
model.ResultsPerPage,
1,
1,
[]index.Document{
{
ID: "book4.epub",
Slug: "james-ellroy-test-d",
Expand All @@ -172,7 +173,7 @@ func testCases() []testCase {
},
},
},
},
),
},
{
"Look for several, not exact terms with multiple leading, trailing and in-between spaces must return a result with all those terms, even if there is something in between",
Expand All @@ -185,11 +186,11 @@ func testCases() []testCase {
Subjects: []string{""},
},
" james ellroy ",
search.PaginatedResult{
Page: 1,
TotalPages: 1,
TotalHits: 1,
Hits: []search.Document{
result.NewPaginated[[]index.Document](
model.ResultsPerPage,
1,
1,
[]index.Document{
{
ID: "book5.epub",
Slug: "james-ellroy-test-e",
Expand All @@ -200,7 +201,7 @@ func testCases() []testCase {
},
},
},
},
),
},
}
}
3 changes: 1 addition & 2 deletions internal/index/bleve_write.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/gosimple/slug"
"github.com/spf13/afero"
"github.com/svera/coreander/v3/internal/metadata"
"github.com/svera/coreander/v3/internal/search"
)

// AddFile adds a file to the index
Expand Down Expand Up @@ -82,7 +81,7 @@ func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error {

func (b *BleveIndexer) createDocument(meta metadata.Metadata, fullPath string, batchSlugs map[string]struct{}) DocumentWrite {
document := DocumentWrite{
Document: search.Document{
Document: Document{
Metadata: meta,
},
SeriesEq: strings.ReplaceAll(slug.Make(meta.Series), "-", ""),
Expand Down
11 changes: 9 additions & 2 deletions internal/index/document.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package index

import "github.com/svera/coreander/v3/internal/search"
import "github.com/svera/coreander/v3/internal/metadata"

type Document struct {
metadata.Metadata
ID string
Slug string
Highlighted bool
}

// DocumentWrite is an extension to Document that is used only when writing to the index,
// as some of its fields are only used to perform searches and not returned
type DocumentWrite struct {
search.Document
Document
AuthorsEq []string
SeriesEq string
SubjectsEq []string
Expand Down
Loading

0 comments on commit b4d85e6

Please sign in to comment.