fix version conflict

Author: miku

cmd/span-crossref-snapshot/main.go

@@ -56,18 +56,18 @@ import (
 	"strings"
 	"sync/atomic"
 
-	"github.com/segmentio/encoding/json"
-
 	"github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 	"github.com/miku/clam"
 	"github.com/miku/span/formats/crossref"
 	"github.com/miku/span/parallel"
 	"github.com/miku/span/xio"
+	"github.com/segmentio/encoding/json"
 	log "github.com/sirupsen/logrus"
 )
 
-// fallback awk script is used, if the filterline executable is not found
+// fallback awk script is used, if the filterline executable is not found;
+// compiled filterline is about 3x faster.
 var fallback = `
 #!/bin/bash
 LIST="$1" LC_ALL=C awk '
@@ -94,6 +94,7 @@ var (
 	verbose           = flag.Bool("verbose", false, "be verbose")
 	pathFile          = flag.String("f", "", "path to a file naming all inputs files to be considered, one file per line")
 	errCountThreshold = flag.Int64("E", 1, "number of json unmarshal errors to tolerate")
+	sortBufferSize    = flag.String("S", "25%", "passed to sort")
 )
 
 // writeFields writes a variable number of values separated by sep to a given
@@ -141,27 +142,8 @@ func main() {
 	)
 	switch {
 	case *pathFile != "":
+		// TODO: this feature would allow to skip the file concatenation step.
 		log.Fatal("not yet implemented")
-		// b, err := ioutil.ReadFile(*pathFile)
-		// if err != nil {
-		// 	log.Fatal(err)
-		// }
-		// s := string(b)
-		// var readers []io.Reader
-		// for _, line := range strings.Split(s, "\n") {
-		// 	line = strings.TrimSpace(line)
-		// 	if len(line) == 0 || strings.HasPrefix(line, "#") {
-		// 		continue
-		// 	}
-		// 	f, err := os.Open(line)
-		// 	if err != nil {
-		// 		log.Fatal(err)
-		// 	}
-		// 	defer f.Close()
-		// 	readers = append(readers, f)
-		// }
-		// log.Printf("path-list: will read from %d files", len(readers))
-		// r = io.MultiReader(readers...)
 	case flag.NArg() == 0:
 		log.Fatal("input file required")
 	case flag.NArg() == 1:
@@ -221,7 +203,7 @@ func main() {
 	var (
 		br      = bufio.NewReader(reader)
 		bw      = bufio.NewWriter(tf)
-		numErrs atomic.Int64
+		numErrs atomic.Int64 // error count across threads
 	)
 	pp := parallel.NewProcessor(br, bw, func(lineno int64, b []byte) ([]byte, error) {
 		var (
@@ -240,13 +222,21 @@ func main() {
 			if n := numErrs.Load(); n > *errCountThreshold {
 				return nil, err
 			} else {
-				log.Printf("skipping error (errs: %d <= max: %d): %v", n, *errCountThreshold, err)
+				log.Printf("skipping error (#err: %d <= max: %d): %v", n, *errCountThreshold, err)
 			}
 			return nil, nil
 		}
 		date, err := doc.Indexed.Date()
 		if err != nil {
-			return nil, err
+			// Encountered with a single document found,
+			// {"DOI":"10.15215\/aupress\/9781897425909.026","score":8.143441}
+			numErrs.Add(1)
+			if n := numErrs.Load(); n > *errCountThreshold {
+				return nil, err
+			} else {
+				log.Printf("skipping error (#err: %d <= max: %d): %v", n, *errCountThreshold, err)
+			}
+			return nil, nil
 		}
 		if _, ok := excludes[doc.DOI]; ok {
 			return nil, nil
@@ -273,7 +263,7 @@ func main() {
 	// Stage 2: Identify relevant records. Sort by DOI (3), then date reversed (2);
 	// then unique by DOI (3). Should keep the entry of the last update (filename,
 	// document date, DOI).
-	fastsort := `LC_ALL=C sort -S20%`
+	fastsort := fmt.Sprintf(`LC_ALL=C sort -S%s`, *sortBufferSize)
 	cmd := `{{ f }} -k3,3 -rk2,2 {{ input }} | {{ f }} -k3,3 -u | cut -f1 | {{ f }} -n > {{ output }}`
 	log.WithFields(log.Fields{
 		"prefix":    "stage 2",

cmd/span-import/main.go

@@ -52,6 +52,7 @@ var (
 	cpuProfile  = flag.String("cpuprofile", "", "write cpu profile to file")
 	memProfile  = flag.String("memprofile", "", "write heap profile to file (go tool pprof -png --alloc_objects program mem.pprof > mem.png)")
 	logfile     = flag.String("logfile", "", "path to logfile to append to, otherwise stderr")
+	verbose     = flag.Bool("verbose", false, "be verbose")
 )
 
 // Factory creates things.
@@ -111,6 +112,9 @@ func processXML(r io.Reader, w io.Writer, name string) error {
 		output, err := converter.ToIntermediateSchema()
 		if err != nil {
 			if _, ok := err.(span.Skip); ok {
+				if *verbose {
+					log.Printf("%v", err)
+				}
 				continue
 			}
 			return err
@@ -138,6 +142,9 @@ func processJSON(r io.Reader, w io.Writer, name string) error {
 		}
 		output, err := converter.ToIntermediateSchema()
 		if _, ok := err.(span.Skip); ok {
+			if *verbose {
+				log.Printf("%v", err)
+			}
 			return nil, nil
 		}
 		if err != nil {
@@ -192,12 +199,10 @@ func processText(r io.Reader, w io.Writer, name string) error {
 
 func main() {
 	flag.Parse()
-
 	if *showVersion {
 		fmt.Println(span.AppVersion)
 		os.Exit(0)
 	}
-
 	if *cpuProfile != "" {
 		f, err := os.Create(*cpuProfile)
 		if err != nil {
@@ -208,7 +213,6 @@ func main() {
 		}
 		defer pprof.StopCPUProfile()
 	}
-
 	if *list {
 		var keys []string
 		for k := range FormatMap {
@@ -220,7 +224,6 @@ func main() {
 		}
 		os.Exit(0)
 	}
-
 	if *logfile != "" {
 		f, err := os.OpenFile(*logfile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 		if err != nil {
@@ -232,12 +235,9 @@ func main() {
 		logger.Out = f
 		log.SetOutput(logger.Writer())
 	}
-
 	w := bufio.NewWriter(os.Stdout)
 	defer w.Flush()
-
 	var reader io.Reader = os.Stdin
-
 	if flag.NArg() > 0 {
 		var files []io.Reader
 		for _, filename := range flag.Args() {
@@ -250,7 +250,6 @@ func main() {
 		}
 		reader = io.MultiReader(files...)
 	}
-
 	switch *name {
 	// XXX: Configure this in one place.
 	case

common.go

@@ -29,7 +29,7 @@ import (
 
 const (
 	// AppVersion of span package. Commandline tools will show this on -v.
-	AppVersion = "0.1.364"
+	AppVersion = "0.1.366"
 	// KeyLengthLimit was a limit imposed by the memcached protocol, which
 	// was used for blob storage until Q1 2017. We switched the key-value
 	// store, so this limit is somewhat obsolete.

docs/span.1

@@ -21,7 +21,7 @@ schema and integration tools
 .PP
 \fB\fCspan\-update\-labels\fR [\fB\fC\-f\fR \fIfile\fP, \fB\fC\-s\fR \fIseparator\fP] < \fIfile\fP
 .PP
-\fB\fCspan\-crossref\-snapshot\fR [\fB\fC\-x\fR \fIfile\fP] \-o \fIfile\fP \fIfile\fP
+\fB\fCspan\-crossref\-snapshot\fR [\fB\fC\-x\fR \fIfile\fP] [\fB\fC\-S\fR \fISIZE\fP] \-o \fIfile\fP \fIfile\fP
 .PP
 \fB\fCspan\-local\-data\fR < \fIfile\fP
 .PP
@@ -37,7 +37,7 @@ schema and integration tools
 .PP
 \fB\fCspan\-crossref\-members\fR [\fB\fC\-base\fR \fIURL\fP] [\fB\fC\-offset\fR \fIN\fP] [\fB\fC\-rows\fR \fIN\fP] [\fB\fC\-q\fR] [\fB\fC\-sleep\fR \fIduration\fP]
 .PP
-\fB\fCspan\-crossref\-sync\fR [\fB\fC\-P\fR \fIprefix\fP] [\fB\fC\-i\fR \fIinterval] [\fB\fC\-p\fR \fIcompress\-program\fP] [\fB\fC\-s\fR \fIdate\fP] [\fB\fC\-e\fR *date\fP]
+\fB\fCspan\-crossref\-sync\fR [\fB\fC\-P\fR \fIprefix\fP] [\fB\fC\-i\fR *interval] [\fB\fC\-p\fR \fIcompress\-program\fP] [\fB\fC\-s\fR \fIdate\fP] [\fB\fC\-e\fR \fIdate\fP] [\fB\fC\-E\fR \fInumerrors\fP]
 .SH DESCRIPTION
 .PP
 The \fB\fCspan\fR tools convert to and from an intermediate schema and support
@@ -63,7 +63,7 @@ EXAMPLE for a CONFIGURATION FILE. \fB\fCspan\-review\fR details in INDEX REVIEW.
 List supported formats. \fB\fCspan\-import\fR, \fB\fCspan\-export\fR only.
 .TP
 \fB\fC\-verbose\fR
-More output. \fB\fCspan\-check\fR only.
+More output. \fB\fCspan\-check\fR, \fB\fCspan\-import\fR only.
 .TP
 \fB\fC\-b\fR \fIN\fP
 Batch size. \fB\fCspan\-tag\fR, \fB\fCspan\-check\fR, \fB\fCspan\-import\fR, \fB\fCspan\-export\fR, \fB\fCspan\-crossref\-snapshot\fR only.
@@ -154,6 +154,12 @@ Suppress logging output, \fB\fCspan\-crossref\-members\fR only.
 .TP
 \fB\fC\-h\fR
 Show usage.
+.TP
+\fB\fC\-E\fR \fInumerrors\fP
+Number of errors to tolerate during processing. \fB\fCspan\-crossref\-snapshot\fR only.
+.TP
+\fB\fC\-S\fR \fIsize\fP
+Buffer size, passed to sort. \fB\fCspan\-crossref\-snapshot\fR only.
 .SH EXAMPLES
 .PP
 List supported formats for conversion to intermediate schema:

docs/span.md

@@ -26,7 +26,7 @@ SYNOPSIS
 
 `span-update-labels` [`-f` *file*, `-s` *separator*] < *file*
 
-`span-crossref-snapshot` [`-x` *file*] -o *file* *file*
+`span-crossref-snapshot` [`-x` *file*] [`-S` *SIZE*] -o *file* *file*
 
 `span-local-data` < *file*
 
@@ -42,7 +42,7 @@ SYNOPSIS
 
 `span-crossref-members` [`-base` *URL*] [`-offset` *N*] [`-rows` *N*] [`-q`] [`-sleep` *duration*]
 
-`span-crossref-sync` [`-P` *prefix*] [`-i` *interval] [`-p` *compress-program*] [`-s` *date*] [`-e` *date*]
+`span-crossref-sync` [`-P` *prefix*] [`-i` *interval] [`-p` *compress-program*] [`-s` *date*] [`-e` *date*] [`-E` *numerrors*]
 
 
 DESCRIPTION
@@ -73,7 +73,7 @@ This section is correct, but incomplete. Consult `-h` for further flags.
   List supported formats. `span-import`, `span-export` only.
 
 `-verbose`
-  More output. `span-check` only.
+  More output. `span-check`, `span-import` only.
 
 `-b` *N*
   Batch size. `span-tag`, `span-check`, `span-import`, `span-export`, `span-crossref-snapshot` only.
@@ -165,6 +165,12 @@ This section is correct, but incomplete. Consult `-h` for further flags.
 `-h`
   Show usage.
 
+`-E` *numerrors*
+  Number of errors to tolerate during processing. `span-crossref-snapshot` only.
+
+`-S` *size*
+  Buffer size, passed to sort. `span-crossref-snapshot` only.
+
 EXAMPLES
 --------
 

freeze.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -25,7 +24,7 @@ func UnfreezeFilterConfig(frozenfile string) (dir, blob string, err error) {
 		mappings = make(map[string]string)
 		b        []byte
 	)
-	if dir, err = ioutil.TempDir("", "span-tag-unfreeze-"); err != nil {
+	if dir, err = os.MkdirTemp("", "span-tag-unfreeze-"); err != nil {
 		return
 	}
 	if r, err = zip.OpenReader(frozenfile); err != nil {
@@ -66,7 +65,7 @@ func UnfreezeFilterConfig(frozenfile string) (dir, blob string, err error) {
 		}
 	}
 	blob = filepath.Join(dir, "blob")
-	if b, err = ioutil.ReadFile(blob); err != nil {
+	if b, err = os.ReadFile(blob); err != nil {
 		return
 	}
 	for url, file := range mappings {
@@ -75,7 +74,7 @@ func UnfreezeFilterConfig(frozenfile string) (dir, blob string, err error) {
 		replacement := []byte(fmt.Sprintf(`"file://%s"`, filepath.Join(dir, file)))
 		b = bytes.Replace(b, value, replacement, -1)
 	}
-	if err = ioutil.WriteFile(blob, b, 0777); err != nil {
+	if err = os.WriteFile(blob, b, 0777); err != nil {
 		return
 	}
 	return dir, blob, nil

packaging/deb/span/DEBIAN/control

@@ -1,5 +1,5 @@
 Package: span
-Version: 0.1.364
+Version: 0.1.366
 Section: utils
 Priority: optional
 Architecture: amd64

packaging/rpm/span.spec

@@ -1,6 +1,6 @@
 Summary:    Library data tools.
 Name:       span
-Version:    0.1.364
+Version:    0.1.366
 Release:    0
 License:    GPL
 ExclusiveArch:  x86_64