add SuppressFormatParameter for debugging endpoints

use case: metha-sync -format epicur -no-intervals -suppress-format-parameter http://123.45.12.34:1234/oai2
miku · Jul 21, 2016 · 24a5828 · 24a5828
1 parent 1069b5d
commit 24a5828
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 19 deletions.
diff --git a/cmd/metha-sync/main.go b/cmd/metha-sync/main.go
@@ -17,6 +17,7 @@ func main() {
 	maxRequests := flag.Int("max", 65536, "maximum number of token loops")
 	disableSelectiveHarvesting := flag.Bool("no-intervals", false, "harvest in one go, for funny endpoints")
 	ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "do not stop on HTTP errors, just skip to the next interval")
+	suppressFormatParameter := flag.Bool("suppress-format-parameter", false, "do not send format parameter")
 	version := flag.Bool("v", false, "show version")
 
 	logFile := flag.String("log", "", "filename to log to")
@@ -67,6 +68,7 @@ func main() {
 	harvest.DisableSelectiveHarvesting = *disableSelectiveHarvesting
 	harvest.MaxEmptyResponses = 10
 	harvest.IgnoreHTTPErrors = *ignoreHTTPErrors
+	harvest.SuppressFormatParameter = *suppressFormatParameter
 
 	log.Printf("harvest: %+v", harvest)
 

diff --git a/harvest.go b/harvest.go
@@ -57,6 +57,7 @@ type Harvest struct {
 	CleanBeforeDecode          bool
 	IgnoreHTTPErrors           bool
 	MaxEmptyResponses          int
+	SuppressFormatParameter    bool
 
 	Identify *Identify
 	Started  time.Time
@@ -271,12 +272,13 @@ func (h *Harvest) runInterval(iv Interval) error {
 
 	for {
 		req := Request{
-			BaseURL:           h.BaseURL,
-			MetadataPrefix:    h.Format,
-			Verb:              "ListRecords",
-			Set:               h.Set,
-			ResumptionToken:   token,
-			CleanBeforeDecode: h.CleanBeforeDecode,
+			BaseURL:                 h.BaseURL,
+			MetadataPrefix:          h.Format,
+			Verb:                    "ListRecords",
+			Set:                     h.Set,
+			ResumptionToken:         token,
+			CleanBeforeDecode:       h.CleanBeforeDecode,
+			SuppressFormatParameter: h.SuppressFormatParameter,
 		}
 
 		var filedate string

diff --git a/request.go b/request.go
@@ -20,15 +20,16 @@ var (
 // A Request can express any request, that can be sent to an OAI server. Not all
 // combination of values will yield valid requests.
 type Request struct {
-	BaseURL           string
-	Verb              string
-	Identifier        string
-	MetadataPrefix    string
-	From              string
-	Until             string
-	Set               string
-	ResumptionToken   string
-	CleanBeforeDecode bool
+	BaseURL                 string
+	Verb                    string
+	Identifier              string
+	MetadataPrefix          string
+	From                    string
+	Until                   string
+	Set                     string
+	ResumptionToken         string
+	CleanBeforeDecode       bool
+	SuppressFormatParameter bool
 }
 
 type Values struct {
@@ -108,8 +109,10 @@ func (r *Request) URL() (*url.URL, error) {
 	switch r.Verb {
 	case "ListMetadataFormats", "ListSets":
 	case "ListIdentifiers", "ListRecords":
-		if err := addRequired("metadataPrefix", r.MetadataPrefix); err != nil {
-			return nil, err
+		if !r.SuppressFormatParameter {
+			if err := addRequired("metadataPrefix", r.MetadataPrefix); err != nil {
+				return nil, err
+			}
 		}
 		addOptional("from", r.From)
 		addOptional("until", r.Until)
@@ -120,8 +123,10 @@ func (r *Request) URL() (*url.URL, error) {
 		if err := addRequired("identifier", r.Identifier); err != nil {
 			return nil, err
 		}
-		if err := addRequired("metadataPrefix", r.MetadataPrefix); err != nil {
-			return nil, err
+		if !r.SuppressFormatParameter {
+			if err := addRequired("metadataPrefix", r.MetadataPrefix); err != nil {
+				return nil, err
+			}
 		}
 	default:
 		return nil, ErrInvalidVerb