diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d7e9636 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM golang +MAINTAINER Sawood Alam + +COPY . /go/src/github.com/oduwsdl/memgator +WORKDIR /go/src/github.com/oduwsdl/memgator +RUN go install -v + +ENTRYPOINT ["memgator"] diff --git a/README.md b/README.md index bb6ff36..b9db2c0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,99 @@ -# memgator -A Memento Aggregator CLI and Server in Go +# MemGator + +A Memento Aggregator CLI and Server in [Go](https://golang.org/). + +## Features + +* The binary (available for various platforms) can be used as the CLI or run as a Web Service +* Results available in three formats - Link/JSON/CDXJ +* TimeMap, TimeGate, TimeNave, and Redirect endpoints +* Good API parity with the [main Memento Aggregator service](http://timetravel.mementoweb.org/guide/api/) +* Concurrent - Splits every session in subtasks for parallel execution +* Parallel - Utilizes all the available CPUs +* Custom archive list (a local JSON file or a remote URL) - a sample JSON is included in the repository +* Probability based archive prioritization and limit +* Three levels of customizable timeouts for greater control over remote requests +* Customizable logging and profiling in CDXJ format +* Customizable endpoint URLs - helpful in load-balancing +* Customizable User-Agent to be sent to each archive +* [CORS](http://www.w3.org/TR/cors/) support to make it easy to use it from JavaScript clients +* Memento count exposed in the header that can be retrieved via `HEAD` request +* [Docker](https://www.docker.com/) friendly - An image available as [ibnesayeed/memgator](https://hub.docker.com/r/ibnesayeed/memgator/) +* Sensible defaults - Batteries included, but replaceable + +## Usage + +### CLI + +Command line interface of MemGator allows retrieval of TimeMap and TimeGate over `STDOUT` in all supported formats. Info/Profiling (in verbose mode) and Error output is available on `STDERR` unless appropriate files are configured. For further details, see the full usage. + +``` +$ memgator [options] {URI-R} # TimeMap CLI +$ memgator [options] {URI-R} {YYYY[MM[DD[hh[mm[ss]]]]]} # TimeGate CLI +``` + +### Server + +When run as a Web Service, MemGator exposes three customizable endpoints as follows: + +``` +$ memgator [options] server +TimeMap : http://localhost:1208/timemap/timemap/link|json|cdxj/{URI-R} +TimeGate : http://localhost:1208/timegate/timegate/{URI-R} [Accept-Datetime] +TimeNav : http://localhost:1208/timenav/link|json|cdxj/{YYYY[MM[DD[hh[mm[ss]]]]]}/{URI-R} +Redirect : http://localhost:1208/redirect/{YYYY[MM[DD[hh[mm[ss]]]]]}/{URI-R} +``` + +The `TimeMap` and `TimeGate` responses are in accordance with the [Memento RFC](http://tools.ietf.org/html/rfc7089). Additionally, the TimeMap endpoint also supports some additional serialization formats. The `TimeNav` service is a URL friendly way to expose the same information in the response body (in various formats) as available in the `Link` header of the `TimeGate` response without the need of a header based time negotiation. The `Redirect` service resolves the datetime (full or partial) passed in the URL and redirects to the closest Memento. + +## Download and Install + +Depending on the machine and operating system download appropriate binary from the releases page. Changed the mode of the file to executable `chmod +x MemGator-BINARY`. Run from the current location of the downloaded binary or rename it to `memgator` and move it into a directory that is in the `PATH` (such as `/usr/local/bin/`). + +## Running as a Docker Container + +``` +$ docker run ibnesayeed/memgator -h +$ docker run ibnesayeed/memgator [options] {URI-R} +$ docker run ibnesayeed/memgator [options] {URI-R} {YYYY[MM[DD[hh[mm[ss]]]]]} +$ docker run ibnesayeed/memgator [options] server +``` + +### Full Usage + +``` +Usage: + memgator [options] {URI-R} # TimeMap CLI + memgator [options] {URI-R} {YYYY[MM[DD[hh[mm[ss]]]]]} # TimeGate CLI + memgator [options] server # Run as a Web Service + +Options: + -A, --agent=MemGator:1.0-beta <{CONTACT}> User-agent string sent to archives + -a, --arcs=http://www.cs.odu.edu/~salam/archives.json Local/remote JSON file path/URL for list of archives + -c, --contact=@WebSciDL Email/URL/Twitter handle - Used in the user-agent + -f, --format=Link Output format - Link/JSON/CDXJ + -g, --timegate=http://{SERVICE}/timegate TimeGate base URL - default based on service URL + -H, --host=localhost Host name - only used in web service mode + -k, --topk=-1 Aggregate only top k archives based on probability + -l, --log= Log file location - Defaults to STDERR + -m, --timemap=http://{SERVICE}/timemap TimeMap base URL - default based on service URL + -P, --profile= Profile file location - Defaults to Logfile + -p, --port=1208 Port number - only used in web service mode + -r, --restimeout=20s Response timeout for each archive + -s, --service=http://{HOST}[:{PORT}] Service base URL - default based on host & port + -T, --hdrtimeout=15s Header timeout for each archive + -t, --contimeout=5s Connection timeout for each archive + -V, --verbose=false Show Info and Profiling messages on STDERR + -v, --version=false Show name and version +``` + +## Build + +Assuming that Git and Go are installed, the `GOPATH` environment variable is set to the Go Workspace directory as described in the [How to Write Go Code](https://golang.org/doc/code.html), and `PATH` includes `$GOPATH/bin`. Cloning, building, and running the code can be done using following commands: + +``` +$ cd $GOPATH +$ go get github.com/oduwsdl/memgator +$ go install github.com/oduwsdl/memgator +$ memgator http://example.com/ +``` diff --git a/archives.json b/archives.json new file mode 100644 index 0000000..ab59974 --- /dev/null +++ b/archives.json @@ -0,0 +1,86 @@ +[ + { + "id": "ia", + "name": "Internet Archive", + "timemap": "http://web.archive.org/web/timemap/link/", + "timegate": "http://web.archive.org/web/", + "probability": 0.5 + }, + { + "id": "proni", + "name": "PRONI Web Archive", + "timemap": "http://webarchive.proni.gov.uk/timemap/", + "timegate": "http://webarchive.proni.gov.uk/timegate/", + "probability": 0.001 + }, + { + "id": "pastpages", + "name": "PastPages Web Archive", + "timemap": "http://www.pastpages.org/timemap/link/", + "timegate": "http://www.pastpages.org/timegate/", + "probability": 0.001 + }, + { + "id": "ba", + "name": "Bibliotheca Alexandrina Web Archive", + "timemap": "http://web.archive.bibalex.org/web/timemap/link/", + "timegate": "http://web.archive.bibalex.org/web/", + "probability": 0.0, + "ignore": true + }, + { + "id": "blarchive", + "name": "UK Web Archive", + "timemap": "http://www.webarchive.org.uk/wayback/archive/timemap/link/", + "timegate": "http://www.webarchive.org.uk/wayback/archive/", + "probability": 0.2 + }, + { + "id": "loc", + "name": "Library of Congress", + "timemap": "http://webarchive.loc.gov/all/timemap/link/", + "timegate": "http://webarchive.loc.gov/all/", + "probability": 0.05 + }, + { + "id": "archiveit", + "name": "Archive-It", + "timemap": "http://wayback.archive-it.org/all/timemap/link/", + "timegate": "http://wayback.archive-it.org/all/", + "probability": 0.1 + }, + { + "id": "ukparliament", + "name": "UK Parliament Web Archive", + "timemap": "http://webarchive.parliament.uk/timemap/", + "timegate": "http://webarchive.parliament.uk/timegate/" + }, + { + "id": "uknationalarchives", + "name": "UK National Archives Web Archive", + "timemap": "http://webarchive.nationalarchives.gov.uk/timemap/", + "timegate": "http://webarchive.nationalarchives.gov.uk/timegate/", + "probability": 0.04 + }, + { + "id": "archive.is", + "name": "archive.today", + "timemap": "http://archive.today/timemap/", + "timegate": "http://archive.today/timegate/", + "probability": 0.07 + }, + { + "id": "is", + "name": "Icelandic Web Archive", + "timemap": "http://wayback.vefsafn.is/wayback/timemap/link/", + "timegate": "http://wayback.vefsafn.is/wayback/", + "probability": 0.06 + }, + { + "id": "swa", + "name": "Stanford Web Archive", + "timemap": "https://swap.stanford.edu/timemap/link/", + "timegate": "https://swap.stanford.edu/", + "probability": 0.001 + } +] diff --git a/main.go b/main.go new file mode 100644 index 0000000..89fa123 --- /dev/null +++ b/main.go @@ -0,0 +1,722 @@ +package main + +import ( + "container/list" + "encoding/json" + "fmt" + flag "github.com/oduwsdl/memgator/pkg/mflag" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "time" +) + +const ( + Name = "MemGator" + Version = "1.0-beta" +) + +var ( + logProfile *log.Logger + logInfo *log.Logger + logError *log.Logger +) + +var format = flag.String([]string{"f", "-format"}, "Link", "Output format - Link/JSON/CDXJ") +var arcsloc = flag.String([]string{"a", "-arcs"}, "http://www.cs.odu.edu/~salam/archives.json", "Local/remote JSON file path/URL for list of archives") +var logfile = flag.String([]string{"l", "-log"}, "", "Log file location - Defaults to STDERR") +var profile = flag.String([]string{"P", "-profile"}, "", "Profile file location - Defaults to Logfile") +var contact = flag.String([]string{"c", "-contact"}, "@WebSciDL", "Email/URL/Twitter handle - Used in the user-agent") +var agent = flag.String([]string{"A", "-agent"}, fmt.Sprintf("%s:%s <{CONTACT}>", Name, Version), "User-agent string sent to archives") +var host = flag.String([]string{"H", "-host"}, "localhost", "Host name - only used in web service mode") +var servicebase = flag.String([]string{"s", "-service"}, "http://{HOST}[:{PORT}]", "Service base URL - default based on host & port") +var mapbase = flag.String([]string{"m", "-timemap"}, "http://{SERVICE}/timemap", "TimeMap base URL - default based on service URL") +var gatebase = flag.String([]string{"g", "-timegate"}, "http://{SERVICE}/timegate", "TimeGate base URL - default based on service URL") +var port = flag.Int([]string{"p", "-port"}, 1208, "Port number - only used in web service mode") +var topk = flag.Int([]string{"k", "-topk"}, -1, "Aggregate only top k archives based on probability") +var verbose = flag.Bool([]string{"V", "-verbose"}, false, "Show Info and Profiling messages on STDERR") +var version = flag.Bool([]string{"v", "-version"}, false, "Show name and version") +var contimeout = flag.Duration([]string{"t", "-contimeout"}, time.Duration(5*time.Second), "Connection timeout for each archive") +var hdrtimeout = flag.Duration([]string{"T", "-hdrtimeout"}, time.Duration(15*time.Second), "Header timeout for each archive") +var restimeout = flag.Duration([]string{"r", "-restimeout"}, time.Duration(20*time.Second), "Response timeout for each archive") + +type Session struct { + Start time.Time +} + +type Archive struct { + Id string `json:"id"` + Name string `json:"name"` + Timemap string `json:"timemap"` + Timegate string `json:"timegate"` + Probability float64 `json:"probability"` + Ignore bool `json:"ignore"` +} + +type Archives []Archive + +func (slice Archives) Len() int { + return len(slice) +} + +func (slice Archives) Less(i, j int) bool { + return slice[i].Probability > slice[j].Probability +} + +func (slice Archives) Swap(i, j int) { + slice[i], slice[j] = slice[j], slice[i] +} + +func (a *Archives) filterIgnored() { + filterPos := 0 + for i := range *a { + if v := (*a)[i]; !v.Ignore { + (*a)[filterPos] = v + filterPos++ + } + } + (*a) = (*a)[:filterPos:filterPos] +} + +var archives Archives + +type Link struct { + Href string + Datetime string + Timeobj time.Time + Timestr string + NavRels []string +} + +var mimeMap = map[string]string{ + "link": "application/link-format", + "json": "application/json", + "cdxj": "application/cdxj+ors", +} + +var regs = map[string]*regexp.Regexp{ + "isprtcl": regexp.MustCompile(`https?://`), + "linkdlm": regexp.MustCompile(`\s*"?\s*,\s*<\s*`), + "attrdlm": regexp.MustCompile(`\s*>?"?\s*;\s*`), + "kvaldlm": regexp.MustCompile(`\s*=\s*"?\s*`), + "memento": regexp.MustCompile(`\bmemento\b`), + "dttmstr": regexp.MustCompile(`^(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?$`), + "tmappth": regexp.MustCompile(`^timemap/(link|json|cdxj)/.+`), + "tgatpth": regexp.MustCompile(`^timegate/.+`), + "tnavpth": regexp.MustCompile(`^timenav/(link|json|cdxj)/(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/.+`), + "rdrcpth": regexp.MustCompile(`^redirect/(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/.+`), +} + +func dialTimeout(network, addr string) (net.Conn, error) { + return net.DialTimeout(network, addr, *contimeout) +} + +var transport = http.Transport{ + Dial: dialTimeout, + ResponseHeaderTimeout: *hdrtimeout, +} + +var client = http.Client{ + Transport: &transport, + Timeout: *restimeout, +} + +func readArchives() (body []byte, err error) { + if !regs["isprtcl"].MatchString(*arcsloc) { + body, err = ioutil.ReadFile(*arcsloc) + return + } + res, err := http.Get(*arcsloc) + if err != nil { + return + } + defer res.Body.Close() + if res.StatusCode != 200 { + err = fmt.Errorf(res.Status) + return + } + body, err = ioutil.ReadAll(res.Body) + return +} + +func splitLinks(lnkrcvd chan string, lnksplt chan string) { + defer close(lnksplt) + lnkstr := <-lnkrcvd + strlen := len(lnkstr) + q := false + i := 0 + j := 0 + for ; j < strlen; j++ { + switch lnkstr[j] { + case '"': + q = !q + case ',': + if !q { + lnksplt <- lnkstr[i:j] + i = j + 1 + } + } + } + if i < j { + lnksplt <- lnkstr[i:j] + } +} + +func extractMementos(lnksplt chan string) (tml *list.List) { + tml = list.New() + for lnk := range lnksplt { + lnk = strings.Trim(lnk, "<\" \t\n\r") + parts := regs["attrdlm"].Split(lnk, -1) + linkmap := map[string]string{"href": parts[0]} + for _, attr := range parts[1:] { + kv := regs["kvaldlm"].Split(attr, 2) + linkmap[kv[0]] = kv[1] + } + dtm, ok := linkmap["datetime"] + if !ok { + continue + } + rel, ok := linkmap["rel"] + if !ok { + continue + } + if !regs["memento"].MatchString(rel) { + continue + } + pdtm, err := time.Parse(time.RFC1123, dtm) + if err != nil { + logError.Printf("Error parsing datetime (%s): %v", dtm, err) + continue + } + link := Link{ + Href: linkmap["href"], + Datetime: dtm, + Timeobj: pdtm, + Timestr: pdtm.Format(time.RFC3339), + } + e := tml.Back() + for ; e != nil; e = e.Prev() { + if link.Timestr > e.Value.(Link).Timestr { + tml.InsertAfter(link, e) + break + } + } + if e == nil { + tml.PushFront(link) + } + } + return +} + +func fetchTimemap(urir string, arch Archive, tmCh chan *list.List, wg *sync.WaitGroup, dttmp *time.Time, sess *Session) { + start := time.Now() + defer wg.Done() + url := arch.Timemap + urir + expst := 200 + if dttmp != nil { + url = arch.Timegate + urir + expst = 302 + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + profileTime(arch.Id, "timemapfetch", fmt.Sprintf("Request error in %s", arch.Name), start, sess) + logError.Printf("%s => Request error: %v", arch.Id, err) + return + } + req.Header.Add("User-Agent", *agent) + var res *http.Response + if dttmp == nil { + res, err = client.Do(req) + } else { + req.Header.Add("Accept-Datetime", dttmp.Format(time.RFC1123)) + res, err = transport.RoundTrip(req) + } + if err != nil { + profileTime(arch.Id, "timemapfetch", fmt.Sprintf("Network error in %s", arch.Name), start, sess) + logError.Printf("%s => Network error: %v", arch.Id, err) + return + } + defer res.Body.Close() + if res.StatusCode != expst { + profileTime(arch.Id, "timemapfetch", fmt.Sprintf("Response error in %s, Stutus: %d", arch.Name, res.StatusCode), start, sess) + logInfo.Printf("%s => Response error: %s (expected: %d)", arch.Id, res.Status, expst) + return + } + lnks := res.Header.Get("Link") + if dttmp == nil { + body, err := ioutil.ReadAll(res.Body) + if err != nil { + profileTime(arch.Id, "timemapfetch", fmt.Sprintf("Response read error in %s", arch.Name), start, sess) + logError.Printf("%s => Response read error: %v", arch.Id, err) + return + } + lnks = string(body) + } + profileTime(arch.Id, "timemapfetch", fmt.Sprintf("TimeMap fethched from %s", arch.Name), start, sess) + start = time.Now() + lnkrcvd := make(chan string, 1) + lnksplt := make(chan string, 128) + lnkrcvd <- lnks + go splitLinks(lnkrcvd, lnksplt) + tml := extractMementos(lnksplt) + tmCh <- tml + profileTime(arch.Id, "extractmementos", fmt.Sprintf("%d Mementos extracted from %s", tml.Len(), arch.Name), start, sess) + logInfo.Printf("%s => Success: %d mementos", arch.Id, tml.Len()) +} + +func serializeLinks(urir string, basetm *list.List, format string, dataCh chan string, navonly bool, sess *Session) { + start := time.Now() + defer profileTime("AGGREGATOR", "serialize", fmt.Sprintf("%d mementos serialized", basetm.Len()), start, sess) + defer close(dataCh) + switch strings.ToLower(format) { + case "link": + dataCh <- fmt.Sprintf(`<%s>; rel="original",`+"\n", urir) + if !navonly { + dataCh <- fmt.Sprintf(`<%s/link/%s>; rel="self"; type="application/link-format",`+"\n", *mapbase, urir) + } + for e := basetm.Front(); e != nil; e = e.Next() { + lnk := e.Value.(Link) + if navonly && lnk.NavRels == nil { + continue + } + rels := "memento" + if lnk.NavRels != nil { + rels = strings.Join(lnk.NavRels, " ") + " " + rels + rels = strings.Replace(rels, "closest ", "", -1) + } + dataCh <- fmt.Sprintf(`<%s>; rel="%s"; datetime="%s",`+"\n", lnk.Href, rels, lnk.Datetime) + } + dataCh <- fmt.Sprintf(`<%s/link/%s>; anchor="%s"; rel="timemap"; type="application/link-format",`+"\n", *mapbase, urir, urir) + dataCh <- fmt.Sprintf(`<%s/json/%s>; anchor="%s"; rel="timemap"; type="application/json",`+"\n", *mapbase, urir, urir) + dataCh <- fmt.Sprintf(`<%s/cdxj/%s>; anchor="%s"; rel="timemap"; type="application/cdxj+ors",`+"\n", *mapbase, urir, urir) + dataCh <- fmt.Sprintf(`<%s/%s>; anchor="%s"; rel="timegate"`+"\n", *gatebase, urir, urir) + case "json": + dataCh <- fmt.Sprintf("{\n"+` "original_uri": "%s",`+"\n"+` "mementos": {`+"\n", urir) + if !navonly { + dataCh <- ` "list": [`+"\n" + } + navs := "" + for e := basetm.Front(); e != nil; e = e.Next() { + lnk := e.Value.(Link) + if navonly && lnk.NavRels == nil { + continue + } + if lnk.NavRels != nil { + for _, rl := range lnk.NavRels { + navs += fmt.Sprintf(` "%s": {`+"\n"+` "datetime": "%s",`+"\n"+` "uri": "%s"`+"\n },\n", rl, lnk.Timestr, lnk.Href) + } + } + if !navonly { + dataCh <- fmt.Sprintf(` {`+"\n"+` "datetime": "%s",`+"\n"+` "uri": "%s"`+"\n }", lnk.Timestr, lnk.Href) + if e.Next() != nil { + dataCh <- ",\n" + } + } + } + if !navonly { + dataCh <- "\n ],\n" + } + dataCh <- strings.TrimRight(navs, ",\n") + dataCh <- fmt.Sprintf("\n },\n"+` "timemap_uri": {`+"\n") + dataCh <- fmt.Sprintf(` "link_format": "%s/link/%s",`+"\n", *mapbase, urir) + dataCh <- fmt.Sprintf(` "json_format": "%s/json/%s",`+"\n", *mapbase, urir) + dataCh <- fmt.Sprintf(` "cdxj_format": "%s/cdxj/%s"`+"\n },\n", *mapbase, urir) + dataCh <- fmt.Sprintf(` "timegate_uri": "%s/%s"`+"\n}\n", *gatebase, urir) + case "cdxj": + dataCh <- fmt.Sprintf(`@meta {"original_uri": "%s"}`+"\n", urir) + dataCh <- fmt.Sprintf(`@meta {"timegate_uri": "%s/%s"}`+"\n", *gatebase, urir) + dataCh <- fmt.Sprintf(`@meta {"timemap_uri": {"link_format": "%s/link/%s", "json_format": "%s/json/%s", "cdxj_format": "%s/cdxj/%s"}`+"\n", *mapbase, urir, *mapbase, urir, *mapbase, urir) + for e := basetm.Front(); e != nil; e = e.Next() { + lnk := e.Value.(Link) + if navonly && lnk.NavRels == nil { + continue + } + rels := "memento" + if lnk.NavRels != nil { + rels = strings.Join(lnk.NavRels, " ") + " " + rels + } + dataCh <- fmt.Sprintf(`%s {"uri": "%s", "rel"="%s", "datetime"="%s"}`+"\n", lnk.Timestr, lnk.Href, rels, lnk.Datetime) + } + default: + dataCh <- fmt.Sprintf("Unrecognized format: %s\n", format) + } +} + +func aggregateTimemap(urir string, dttmp *time.Time, sess *Session) (basetm *list.List) { + var wg sync.WaitGroup + tmCh := make(chan *list.List, len(archives)) + for i, arch := range archives { + if i == *topk { + break + } + wg.Add(1) + go fetchTimemap(urir, arch, tmCh, &wg, dttmp, sess) + } + go func() { + wg.Wait() + close(tmCh) + }() + basetm = list.New() + for newtm := range tmCh { + start := time.Now() + if newtm.Len() == 0 { + continue + } + if basetm.Len() == 0 { + basetm = newtm + continue + } + if newtm.Len() > basetm.Len() { + newtm, basetm = basetm, newtm + } + m := basetm.Back() + e := newtm.Back() + for e != nil { + if m != nil { + if e.Value.(Link).Timestr > m.Value.(Link).Timestr { + basetm.InsertAfter(e.Value, m) + e = e.Prev() + } else { + m = m.Prev() + } + } else { + for e != nil { + basetm.PushFront(e.Value) + e = e.Prev() + } + } + } + profileTime("AGGREGATOR", "aggregate", fmt.Sprintf("%d Mementos accumulated and sorted", basetm.Len()), start, sess) + } + return +} + +func parseUri(uri string) (urir string, err error) { + u, err := url.Parse(uri) + if err != nil { + logError.Printf("URI parsing error (%s): %v", uri, err) + return + } + if u.Scheme == "" { + urir = "http://" + uri + } + urir = uri + return +} + +func paddedTime(dttmstr string) (dttm *time.Time, err error) { + m := regs["dttmstr"].FindStringSubmatch(dttmstr) + dts := m[1] + dts += (m[2] + "01")[:2] + dts += (m[3] + "01")[:2] + dts += (m[4] + "00")[:2] + dts += (m[5] + "00")[:2] + dts += (m[6] + "00")[:2] + var dtm time.Time + dtm, err = time.Parse("20060102150405", dts) + if err != nil { + logError.Printf("Time parsing error (%s): %v", dttmstr, err) + } + dttm = &dtm + return +} + +func profileTime(origin string, role string, info string, start time.Time, sess *Session) { + end := time.Now() + begin := sess.Start.UnixNano() + info += fmt.Sprintf(" - Duration: %v", end.Sub(start)) + logProfile.Printf(`%d {"origin": "%s", "role": "%s", "info": "%s", "start": %d, "end": %d}`, begin, origin, role, info, start.UnixNano()-begin, end.UnixNano()-begin) +} + +func setNavRels(basetm *list.List, dttmp *time.Time, sess *Session) (navonly bool, closest string) { + start := time.Now() + defer profileTime("AGGREGATOR", "setnav", "Navigational relations annotated", start, sess) + itm := basetm.Front().Value.(Link) + itm.NavRels = append(itm.NavRels, "first") + basetm.Front().Value = itm + itm = basetm.Back().Value.(Link) + itm.NavRels = append(itm.NavRels, "last") + basetm.Back().Value = itm + closelm := basetm.Front() + if dttmp != nil { + navonly = true + dttm := *dttmp + etm := closelm.Value.(Link).Timeobj + dur := etm.Sub(dttm) + if dttm.After(etm) { + dur = dttm.Sub(etm) + } + mindur := dur + for e := closelm.Next(); e != nil; e = e.Next() { + etm = e.Value.(Link).Timeobj + dur = etm.Sub(dttm) + if dttm.After(etm) { + dur = dttm.Sub(etm) + } + if dur > mindur { + break + } + mindur = dur + closelm = e + } + itm = closelm.Value.(Link) + closest = itm.Href + itm.NavRels = append(itm.NavRels, "closest") + closelm.Value = itm + if closelm.Next() != nil { + itm = closelm.Next().Value.(Link) + itm.NavRels = append(itm.NavRels, "next") + closelm.Next().Value = itm + } + if closelm.Prev() != nil { + itm = closelm.Prev().Value.(Link) + itm.NavRels = append(itm.NavRels, "prev") + closelm.Prev().Value = itm + } + } + return +} + +func memgatorCli(urir string, format string, dttmp *time.Time) { + start := time.Now() + sess := new(Session) + sess.Start = start + defer profileTime("SESSION", "session", "Complete session", start, sess) + profileTime("AGGREGATOR", "createsess", "Session created", start, sess) + logInfo.Printf("Aggregating Mementos for %s", urir) + basetm := aggregateTimemap(urir, dttmp, sess) + if basetm.Len() == 0 { + return + } + navonly, _ := setNavRels(basetm, dttmp, sess) + dataCh := make(chan string, 1) + go serializeLinks(urir, basetm, format, dataCh, navonly, sess) + for dt := range dataCh { + fmt.Printf(dt) + } + logInfo.Printf("Total Mementos: %d in %s", basetm.Len(), time.Since(start)) +} + +func memgatorService(w http.ResponseWriter, r *http.Request, urir string, format string, dttmp *time.Time) { + start := time.Now() + sess := new(Session) + sess.Start = start + defer profileTime("SESSION", "setnav", "Complete session", start, sess) + profileTime("AGGREGATOR", "createsess", "Session created", start, sess) + logInfo.Printf("Aggregating Mementos for %s", urir) + basetm := aggregateTimemap(urir, dttmp, sess) + w.Header().Set("X-Origin", Name+":"+Version) + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Expose-Headers", "Link, Location, X-Memento-Count, X-Origin") + if basetm.Len() == 0 { + http.NotFound(w, r) + return + } + navonly, closest := setNavRels(basetm, dttmp, sess) + if format == "redirect" { + http.Redirect(w, r, closest, http.StatusFound) + return + } + dataCh := make(chan string, 1) + if format == "timegate" { + go serializeLinks(urir, basetm, "link", dataCh, navonly, sess) + lnkhdr := "" + for dt := range dataCh { + lnkhdr += dt + } + lnkhdr = strings.Replace(lnkhdr, "\n", " ", -1) + w.Header().Set("Link", lnkhdr) + http.Redirect(w, r, closest, http.StatusFound) + return + } + go serializeLinks(urir, basetm, format, dataCh, navonly, sess) + mime, ok := mimeMap[strings.ToLower(format)] + if ok { + w.Header().Set("Content-Type", mime) + } + w.Header().Set("X-Memento-Count", fmt.Sprintf("%d", basetm.Len())) + for dt := range dataCh { + fmt.Fprintf(w, dt) + } + logInfo.Printf("Total Mementos: %d in %s", basetm.Len(), time.Since(start)) +} + +func welcome(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, serviceInfo()) +} + +func router(w http.ResponseWriter, r *http.Request) { + var format, urir, rawuri, rawdtm string + var dttm *time.Time + var err error + requri := r.URL.RequestURI()[1:] + switch { + case regs["tmappth"].MatchString(requri): + p := strings.SplitN(requri, "/", 3) + format = p[1] + rawuri = p[2] + case regs["tnavpth"].MatchString(requri): + p := strings.SplitN(requri, "/", 4) + format = p[1] + rawdtm = p[2] + rawuri = p[3] + case regs["rdrcpth"].MatchString(requri): + p := strings.SplitN(requri, "/", 3) + format = p[0] + rawdtm = p[1] + rawuri = p[2] + case regs["tgatpth"].MatchString(requri): + p := strings.SplitN(requri, "/", 2) + format = p[0] + rawuri = p[1] + gttm := time.Now().UTC() + if hdtm := r.Header.Get("Accept-Datetime"); hdtm != "" { + gttm, err = time.Parse(time.RFC1123, hdtm) + if err != nil { + logError.Printf("Error parsing datetime (%s): %v", hdtm, err) + http.Error(w, "Malformed datetime: "+hdtm, http.StatusBadRequest) + return + } + } + dttm = >tm + default: + http.DefaultServeMux.ServeHTTP(w, r) + return + } + urir, err = parseUri(rawuri) + if err != nil { + http.Error(w, "Malformed URI-R: "+rawuri, http.StatusBadRequest) + return + } + if rawdtm != "" { + dttm, err = paddedTime(rawdtm) + if err != nil { + http.Error(w, "Malformed datetime: "+rawdtm, http.StatusBadRequest) + return + } + } + memgatorService(w, r, urir, format, dttm) +} + +func overrideFlags() { + if *agent == fmt.Sprintf("%s:%s <{CONTACT}>", Name, Version) { + *agent = fmt.Sprintf("%s:%s <%s>", Name, Version, *contact) + } + if *servicebase == "http://{HOST}[:{PORT}]" { + if *port == 80 { + *servicebase = fmt.Sprintf("http://%s", *host) + } else { + *servicebase = fmt.Sprintf("http://%s:%d", *host, *port) + } + } + if *mapbase == "http://{SERVICE}/timemap" { + *mapbase = fmt.Sprintf("%s/timemap", *servicebase) + } + if *gatebase == "http://{SERVICE}/timegate" { + *gatebase = fmt.Sprintf("%s/timegate", *servicebase) + } +} + +func serviceInfo() (msg string) { + return fmt.Sprintf("TimeMap : %s/timemap/link|json|cdxj/{URI-R}\nTimeGate : %s/timegate/{URI-R} [Accept-Datetime]\nTimeNav : %s/timenav/link|json|cdxj/{YYYY[MM[DD[hh[mm[ss]]]]]}/{URI-R}\nRedirect : %s/redirect/{YYYY[MM[DD[hh[mm[ss]]]]]}/{URI-R}\n", *mapbase, *gatebase, *servicebase, *servicebase) +} + +func usage() { + fmt.Fprintf(os.Stderr, "\nUsage:\n") + fmt.Fprintf(os.Stderr, " %s [options] {URI-R} # TimeMap CLI\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {URI-R} {YYYY[MM[DD[hh[mm[ss]]]]]} # TimeGate CLI\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] server # Run as a Web Service\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\nOptions:\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\n") +} + +func initLoggers() { + errorHandle := os.Stderr + infoHandle := ioutil.Discard + profileHandle := ioutil.Discard + if *logfile != "" { + lgf, err := os.OpenFile(*logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening log file (%s): %v\n", *logfile, err) + os.Exit(1) + } + errorHandle = lgf + infoHandle = lgf + profileHandle = lgf + } + if *profile != "" { + prf, err := os.OpenFile(*profile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening profile file (%s): %v\n", *profile, err) + os.Exit(1) + } + profileHandle = prf + } + if *verbose { + errorHandle = os.Stderr + infoHandle = os.Stderr + profileHandle = os.Stderr + } + logError = log.New(errorHandle, "ERROR: ", log.Ldate|log.Lmicroseconds|log.Lshortfile) + logInfo = log.New(infoHandle, "INFO: ", log.Ldate|log.Lmicroseconds) + logProfile = log.New(profileHandle, "PROFILE: ", log.Ldate|log.Lmicroseconds) +} + +func main() { + start := time.Now() + flag.Usage = usage + flag.Parse() + overrideFlags() + if *version { + fmt.Printf("%s %s\n", Name, Version) + os.Exit(1) + } + target := flag.Arg(0) + if target == "" { + flag.Usage() + os.Exit(1) + } + initLoggers() + logInfo.Printf("Initializing %s:%s...", Name, Version) + body, err := readArchives() + if err != nil { + fmt.Fprintf(os.Stderr, "Error reading list of archives (%s): %s\n", *arcsloc, err) + os.Exit(1) + } + err = json.Unmarshal(body, &archives) + archives.filterIgnored() + sort.Sort(archives) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing JSON (%s): %s\n", *arcsloc, err) + os.Exit(1) + } + if target == "server" { + addr := fmt.Sprintf(":%d", *port) + fmt.Printf(serviceInfo()) + http.HandleFunc("/", welcome) + http.ListenAndServe(addr, http.HandlerFunc(router)) + } else { + urir, err := parseUri(target) + if err != nil { + os.Exit(1) + } + var dttm *time.Time + if regs["dttmstr"].MatchString(flag.Arg(1)) { + dttm, err = paddedTime(flag.Arg(1)) + if err != nil { + os.Exit(1) + } + } + memgatorCli(urir, *format, dttm) + } + elapsed := time.Since(start) + logInfo.Printf("Uptime: %s", elapsed) +} diff --git a/memgator_logo.png b/memgator_logo.png new file mode 100644 index 0000000..0bed737 Binary files /dev/null and b/memgator_logo.png differ diff --git a/pkg/mflag/flag.go b/pkg/mflag/flag.go new file mode 100644 index 0000000..36d87f6 --- /dev/null +++ b/pkg/mflag/flag.go @@ -0,0 +1,1223 @@ +// Copyright 2014-2015 The Docker & Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package flag implements command-line flag parsing. + + Usage: + + Define flags using flag.String(), Bool(), Int(), etc. + + This declares an integer flag, -f or --flagname, stored in the pointer ip, with type *int. + import "flag /github.com/docker/docker/pkg/mflag" + var ip = flag.Int([]string{"f", "-flagname"}, 1234, "help message for flagname") + If you like, you can bind the flag to a variable using the Var() functions. + var flagvar int + func init() { + // -flaghidden will work, but will be hidden from the usage + flag.IntVar(&flagvar, []string{"f", "#flaghidden", "-flagname"}, 1234, "help message for flagname") + } + Or you can create custom flags that satisfy the Value interface (with + pointer receivers) and couple them to flag parsing by + flag.Var(&flagVal, []string{"name"}, "help message for flagname") + For such flags, the default value is just the initial value of the variable. + + You can also add "deprecated" flags, they are still usable, but are not shown + in the usage and will display a warning when you try to use them. `#` before + an option means this option is deprecated, if there is an following option + without `#` ahead, then that's the replacement, if not, it will just be removed: + var ip = flag.Int([]string{"#f", "#flagname", "-flagname"}, 1234, "help message for flagname") + this will display: `Warning: '-f' is deprecated, it will be replaced by '--flagname' soon. See usage.` or + this will display: `Warning: '-flagname' is deprecated, it will be replaced by '--flagname' soon. See usage.` + var ip = flag.Int([]string{"f", "#flagname"}, 1234, "help message for flagname") + will display: `Warning: '-flagname' is deprecated, it will be removed soon. See usage.` + so you can only use `-f`. + + You can also group one letter flags, bif you declare + var v = flag.Bool([]string{"v", "-verbose"}, false, "help message for verbose") + var s = flag.Bool([]string{"s", "-slow"}, false, "help message for slow") + you will be able to use the -vs or -sv + + After all flags are defined, call + flag.Parse() + to parse the command line into the defined flags. + + Flags may then be used directly. If you're using the flags themselves, + they are all pointers; if you bind to variables, they're values. + fmt.Println("ip has value ", *ip) + fmt.Println("flagvar has value ", flagvar) + + After parsing, the arguments after the flag are available as the + slice flag.Args() or individually as flag.Arg(i). + The arguments are indexed from 0 through flag.NArg()-1. + + Command line flag syntax: + -flag + -flag=x + -flag="x" + -flag='x' + -flag x // non-boolean flags only + One or two minus signs may be used; they are equivalent. + The last form is not permitted for boolean flags because the + meaning of the command + cmd -x * + will change if there is a file called 0, false, etc. You must + use the -flag=false form to turn off a boolean flag. + + Flag parsing stops just before the first non-flag argument + ("-" is a non-flag argument) or after the terminator "--". + + Integer flags accept 1234, 0664, 0x1234 and may be negative. + Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False. + Duration flags accept any input valid for time.ParseDuration. + + The default set of command-line flags is controlled by + top-level functions. The FlagSet type allows one to define + independent sets of flags, such as to implement subcommands + in a command-line interface. The methods of FlagSet are + analogous to the top-level functions for the command-line + flag set. +*/ +package mflag + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" +) + +// ErrHelp is the error returned if the flag -help is invoked but no such flag is defined. +var ErrHelp = errors.New("flag: help requested") + +// ErrRetry is the error returned if you need to try letter by letter +var ErrRetry = errors.New("flag: retry") + +// -- bool Value +type boolValue bool + +func newBoolValue(val bool, p *bool) *boolValue { + *p = val + return (*boolValue)(p) +} + +func (b *boolValue) Set(s string) error { + v, err := strconv.ParseBool(s) + *b = boolValue(v) + return err +} + +func (b *boolValue) Get() interface{} { return bool(*b) } + +func (b *boolValue) String() string { return fmt.Sprintf("%v", *b) } + +func (b *boolValue) IsBoolFlag() bool { return true } + +// optional interface to indicate boolean flags that can be +// supplied without "=value" text +type boolFlag interface { + Value + IsBoolFlag() bool +} + +// -- int Value +type intValue int + +func newIntValue(val int, p *int) *intValue { + *p = val + return (*intValue)(p) +} + +func (i *intValue) Set(s string) error { + v, err := strconv.ParseInt(s, 0, 64) + *i = intValue(v) + return err +} + +func (i *intValue) Get() interface{} { return int(*i) } + +func (i *intValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- int64 Value +type int64Value int64 + +func newInt64Value(val int64, p *int64) *int64Value { + *p = val + return (*int64Value)(p) +} + +func (i *int64Value) Set(s string) error { + v, err := strconv.ParseInt(s, 0, 64) + *i = int64Value(v) + return err +} + +func (i *int64Value) Get() interface{} { return int64(*i) } + +func (i *int64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- uint Value +type uintValue uint + +func newUintValue(val uint, p *uint) *uintValue { + *p = val + return (*uintValue)(p) +} + +func (i *uintValue) Set(s string) error { + v, err := strconv.ParseUint(s, 0, 64) + *i = uintValue(v) + return err +} + +func (i *uintValue) Get() interface{} { return uint(*i) } + +func (i *uintValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- uint64 Value +type uint64Value uint64 + +func newUint64Value(val uint64, p *uint64) *uint64Value { + *p = val + return (*uint64Value)(p) +} + +func (i *uint64Value) Set(s string) error { + v, err := strconv.ParseUint(s, 0, 64) + *i = uint64Value(v) + return err +} + +func (i *uint64Value) Get() interface{} { return uint64(*i) } + +func (i *uint64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- string Value +type stringValue string + +func newStringValue(val string, p *string) *stringValue { + *p = val + return (*stringValue)(p) +} + +func (s *stringValue) Set(val string) error { + *s = stringValue(val) + return nil +} + +func (s *stringValue) Get() interface{} { return string(*s) } + +func (s *stringValue) String() string { return fmt.Sprintf("%s", *s) } + +// -- float64 Value +type float64Value float64 + +func newFloat64Value(val float64, p *float64) *float64Value { + *p = val + return (*float64Value)(p) +} + +func (f *float64Value) Set(s string) error { + v, err := strconv.ParseFloat(s, 64) + *f = float64Value(v) + return err +} + +func (f *float64Value) Get() interface{} { return float64(*f) } + +func (f *float64Value) String() string { return fmt.Sprintf("%v", *f) } + +// -- time.Duration Value +type durationValue time.Duration + +func newDurationValue(val time.Duration, p *time.Duration) *durationValue { + *p = val + return (*durationValue)(p) +} + +func (d *durationValue) Set(s string) error { + v, err := time.ParseDuration(s) + *d = durationValue(v) + return err +} + +func (d *durationValue) Get() interface{} { return time.Duration(*d) } + +func (d *durationValue) String() string { return (*time.Duration)(d).String() } + +// Value is the interface to the dynamic value stored in a flag. +// (The default value is represented as a string.) +// +// If a Value has an IsBoolFlag() bool method returning true, +// the command-line parser makes -name equivalent to -name=true +// rather than using the next command-line argument. +type Value interface { + String() string + Set(string) error +} + +// Getter is an interface that allows the contents of a Value to be retrieved. +// It wraps the Value interface, rather than being part of it, because it +// appeared after Go 1 and its compatibility rules. All Value types provided +// by this package satisfy the Getter interface. +type Getter interface { + Value + Get() interface{} +} + +// ErrorHandling defines how to handle flag parsing errors. +type ErrorHandling int + +const ( + ContinueOnError ErrorHandling = iota + ExitOnError + PanicOnError +) + +// A FlagSet represents a set of defined flags. The zero value of a FlagSet +// has no name and has ContinueOnError error handling. +type FlagSet struct { + // Usage is the function called when an error occurs while parsing flags. + // The field is a function (not a method) that may be changed to point to + // a custom error handler. + Usage func() + + name string + parsed bool + actual map[string]*Flag + formal map[string]*Flag + args []string // arguments after flags + errorHandling ErrorHandling + output io.Writer // nil means stderr; use Out() accessor + nArgRequirements []nArgRequirement + + // Original options to extend flag help message output + examples map[string]string // examples are shown when promping usage + synopsis string // simple usage of command + description string // command description +} + +// A Flag represents the state of a flag. +type Flag struct { + Names []string // name as it appears on command line + Usage string // help message + Value Value // value as set + DefValue string // default value (as text); for usage message +} + +type flagSlice []string + +func (p flagSlice) Len() int { return len(p) } +func (p flagSlice) Less(i, j int) bool { + pi, pj := strings.TrimPrefix(p[i], "-"), strings.TrimPrefix(p[j], "-") + lpi, lpj := strings.ToLower(pi), strings.ToLower(pj) + if lpi != lpj { + return lpi < lpj + } + return pi < pj +} +func (p flagSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// sortFlags returns the flags as a slice in lexicographical sorted order. +func sortFlags(flags map[string]*Flag) []*Flag { + var list flagSlice + + // The sorted list is based on the first name, when flag map might use the other names. + nameMap := make(map[string]string) + + for n, f := range flags { + fName := strings.TrimPrefix(f.Names[0], "#") + nameMap[fName] = n + if len(f.Names) == 1 { + list = append(list, fName) + continue + } + + found := false + for _, name := range list { + if name == fName { + found = true + break + } + } + if !found { + list = append(list, fName) + } + } + sort.Sort(list) + result := make([]*Flag, len(list)) + for i, name := range list { + result[i] = flags[nameMap[name]] + } + return result +} + +// Name returns the name of the FlagSet. +func (f *FlagSet) Name() string { + return f.name +} + +// Out returns the destination for usage and error messages. +func (f *FlagSet) Out() io.Writer { + if f.output == nil { + return os.Stderr + } + return f.output +} + +// SetOutput sets the destination for usage and error messages. +// If output is nil, os.Stderr is used. +func (f *FlagSet) SetOutput(output io.Writer) { + f.output = output +} + +// AddExample adds example for usage. +func (f *FlagSet) AddUsageExample(cmd, desc string) { + // Initialize if size is zero + if len(f.examples) == 0 { + f.examples = make(map[string]string) + } + f.examples[cmd] = desc +} + +// SetSynopsis sets flag description for usage. +func (f *FlagSet) SetUsageSynopsis(synopsis string) { + f.synopsis = synopsis +} + +// SetDescription sets flag description for usage. +func (f *FlagSet) SetUsageDescription(desc string) { + f.description = desc +} + +// VisitAll visits the flags in lexicographical order, calling fn for each. +// It visits all flags, even those not set. +func (f *FlagSet) VisitAll(fn func(*Flag)) { + for _, flag := range sortFlags(f.formal) { + fn(flag) + } +} + +// VisitAll visits the command-line flags in lexicographical order, calling +// fn for each. It visits all flags, even those not set. +func VisitAll(fn func(*Flag)) { + CommandLine.VisitAll(fn) +} + +// Visit visits the flags in lexicographical order, calling fn for each. +// It visits only those flags that have been set. +func (f *FlagSet) Visit(fn func(*Flag)) { + for _, flag := range sortFlags(f.actual) { + fn(flag) + } +} + +// Visit visits the command-line flags in lexicographical order, calling fn +// for each. It visits only those flags that have been set. +func Visit(fn func(*Flag)) { + CommandLine.Visit(fn) +} + +// Lookup returns the Flag structure of the named flag, returning nil if none exists. +func (f *FlagSet) Lookup(name string) *Flag { + return f.formal[name] +} + +// Indicates whether the specified flag was specified at all on the cmd line +func (f *FlagSet) IsSet(name string) bool { + return f.actual[name] != nil +} + +// Lookup returns the Flag structure of the named command-line flag, +// returning nil if none exists. +func Lookup(name string) *Flag { + return CommandLine.formal[name] +} + +// Indicates whether the specified flag was specified at all on the cmd line +func IsSet(name string) bool { + return CommandLine.IsSet(name) +} + +type nArgRequirementType int + +// Indicator used to pass to BadArgs function +const ( + Exact nArgRequirementType = iota + Max + Min +) + +type nArgRequirement struct { + Type nArgRequirementType + N int +} + +// Require adds a requirement about the number of arguments for the FlagSet. +// The first parameter can be Exact, Max, or Min to respectively specify the exact, +// the maximum, or the minimal number of arguments required. +// The actual check is done in FlagSet.CheckArgs(). +func (f *FlagSet) Require(nArgRequirementType nArgRequirementType, nArg int) { + f.nArgRequirements = append(f.nArgRequirements, nArgRequirement{nArgRequirementType, nArg}) +} + +// CheckArgs uses the requirements set by FlagSet.Require() to validate +// the number of arguments. If the requirements are not met, +// an error message string is returned. +func (f *FlagSet) CheckArgs() (message string) { + for _, req := range f.nArgRequirements { + var arguments string + if req.N == 1 { + arguments = "1 argument" + } else { + arguments = fmt.Sprintf("%d arguments", req.N) + } + + str := func(kind string) string { + return fmt.Sprintf("%q requires %s%s", f.name, kind, arguments) + } + + switch req.Type { + case Exact: + if f.NArg() != req.N { + return str("") + } + case Max: + if f.NArg() > req.N { + return str("a maximum of ") + } + case Min: + if f.NArg() < req.N { + return str("a minimum of ") + } + } + } + return "" +} + +// Set sets the value of the named flag. +func (f *FlagSet) Set(name, value string) error { + flag, ok := f.formal[name] + if !ok { + return fmt.Errorf("no such flag -%v", name) + } + if err := flag.Value.Set(value); err != nil { + return err + } + if f.actual == nil { + f.actual = make(map[string]*Flag) + } + f.actual[name] = flag + return nil +} + +// Set sets the value of the named command-line flag. +func Set(name, value string) error { + return CommandLine.Set(name, value) +} + +// PrintDefaults prints, to standard error unless configured +// otherwise, the default values of all defined flags in the set. +func (f *FlagSet) PrintDefaults() { + writer := tabwriter.NewWriter(f.Out(), 20, 1, 4, ' ', 0) + + f.VisitAll(func(flag *Flag) { + format := " -%s=%s" + names := []string{} + for _, name := range flag.Names { + if name[0] != '#' { + names = append(names, name) + } + } + if len(names) > 0 { + val := flag.DefValue + fmt.Fprintf(writer, format, strings.Join(names, ", -"), val) + + // Show usage with fixed length + br := bufio.NewReader(strings.NewReader(flag.Usage)) + for { + buf := make([]byte, 45) + n, err := br.Read(buf) + if err != nil { + break + } + + if n < 1 { + break + } + + // Read until first space + left, _ := br.ReadBytes(" "[0]) + exbuf := append(buf, left...) + + // Delete new-line + output := strings.Replace(string(exbuf), "\n", "", -1) + + // Delete front space + output = strings.TrimLeft(output, " ") + + // Write output to writer + fmt.Fprintln(writer, "\t", output) + } + + //fmt.Fprintln(writer, "\t") + + } + }) + writer.Flush() +} + +func (f *FlagSet) PrintExamples() { + writer := tabwriter.NewWriter(f.Out(), 20, 1, 4, ' ', 0) + for cmd, desc := range f.examples { + fmt.Fprintf(writer, " %s", cmd) + + // Show description with fixed lenge + br := bufio.NewReader(strings.NewReader(desc)) + for { + buf := make([]byte, 45) + n, err := br.Read(buf) + if err != nil { + break + } + + if n < 1 { + break + } + + // Read until first space + left, _ := br.ReadBytes(" "[0]) + exbuf := append(buf, left...) + + // Delete new-line + output := strings.Replace(string(exbuf), "\n", "", -1) + + // Delete front space + output = strings.TrimLeft(output, " ") + + // Write output to writer + fmt.Fprintln(writer, "\t", output) + } + + fmt.Fprintln(writer, "\t") + + } + writer.Flush() +} + +// PrintDefaults prints to standard error the default values of all defined command-line flags. +func PrintDefaults() { + CommandLine.PrintDefaults() +} + +// defaultUsage is the default function to print a usage message. +func defaultUsage(f *FlagSet) { + + if f.synopsis != "" { + fmt.Fprintf(f.Out(), "Usage:\n\n") + fmt.Fprintf(f.Out(), " %s\n\n", f.synopsis) + } + + if f.description != "" { + fmt.Fprintf(f.Out(), "Description:\n\n") + fmt.Fprintf(f.Out(), " %s\n\n", f.description) + } + + fmt.Fprintf(f.Out(), "Options:\n\n") + f.PrintDefaults() + + if len(f.examples) > 0 { + fmt.Fprintf(f.Out(), "Examples:\n\n") + f.PrintExamples() + } +} + +// NOTE: Usage is not just defaultUsage(CommandLine) +// because it serves (via godoc flag Usage) as the example +// for how to write your own usage function. + +// Usage prints to standard error a usage message documenting all defined command-line flags. +// The function is a variable that may be changed to point to a custom function. +var Usage = func() { + fmt.Fprintf(CommandLine.output, "Usage of %s:\n", os.Args[0]) + PrintDefaults() +} + +// FlagCount returns the number of flags that have been defined. +func (f *FlagSet) FlagCount() int { return len(sortFlags(f.formal)) } + +// FlagCountUndeprecated returns the number of undeprecated flags that have been defined. +func (f *FlagSet) FlagCountUndeprecated() int { + count := 0 + for _, flag := range sortFlags(f.formal) { + for _, name := range flag.Names { + if name[0] != '#' { + count++ + break + } + } + } + return count +} + +// NFlag returns the number of flags that have been set. +func (f *FlagSet) NFlag() int { return len(f.actual) } + +// NFlag returns the number of command-line flags that have been set. +func NFlag() int { return len(CommandLine.actual) } + +// Arg returns the i'th argument. Arg(0) is the first remaining argument +// after flags have been processed. +func (f *FlagSet) Arg(i int) string { + if i < 0 || i >= len(f.args) { + return "" + } + return f.args[i] +} + +// Arg returns the i'th command-line argument. Arg(0) is the first remaining argument +// after flags have been processed. +func Arg(i int) string { + return CommandLine.Arg(i) +} + +// NArg is the number of arguments remaining after flags have been processed. +func (f *FlagSet) NArg() int { return len(f.args) } + +// NArg is the number of arguments remaining after flags have been processed. +func NArg() int { return len(CommandLine.args) } + +// Args returns the non-flag arguments. +func (f *FlagSet) Args() []string { return f.args } + +// Args returns the non-flag command-line arguments. +func Args() []string { return CommandLine.args } + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func (f *FlagSet) BoolVar(p *bool, names []string, value bool, usage string) { + f.Var(newBoolValue(value, p), names, usage) +} + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func BoolVar(p *bool, names []string, value bool, usage string) { + CommandLine.Var(newBoolValue(value, p), names, usage) +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func (f *FlagSet) Bool(names []string, value bool, usage string) *bool { + p := new(bool) + f.BoolVar(p, names, value, usage) + return p +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func Bool(names []string, value bool, usage string) *bool { + return CommandLine.Bool(names, value, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func (f *FlagSet) IntVar(p *int, names []string, value int, usage string) { + f.Var(newIntValue(value, p), names, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func IntVar(p *int, names []string, value int, usage string) { + CommandLine.Var(newIntValue(value, p), names, usage) +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func (f *FlagSet) Int(names []string, value int, usage string) *int { + p := new(int) + f.IntVar(p, names, value, usage) + return p +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func Int(names []string, value int, usage string) *int { + return CommandLine.Int(names, value, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func (f *FlagSet) Int64Var(p *int64, names []string, value int64, usage string) { + f.Var(newInt64Value(value, p), names, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func Int64Var(p *int64, names []string, value int64, usage string) { + CommandLine.Var(newInt64Value(value, p), names, usage) +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func (f *FlagSet) Int64(names []string, value int64, usage string) *int64 { + p := new(int64) + f.Int64Var(p, names, value, usage) + return p +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func Int64(names []string, value int64, usage string) *int64 { + return CommandLine.Int64(names, value, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func (f *FlagSet) UintVar(p *uint, names []string, value uint, usage string) { + f.Var(newUintValue(value, p), names, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func UintVar(p *uint, names []string, value uint, usage string) { + CommandLine.Var(newUintValue(value, p), names, usage) +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func (f *FlagSet) Uint(names []string, value uint, usage string) *uint { + p := new(uint) + f.UintVar(p, names, value, usage) + return p +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func Uint(names []string, value uint, usage string) *uint { + return CommandLine.Uint(names, value, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func (f *FlagSet) Uint64Var(p *uint64, names []string, value uint64, usage string) { + f.Var(newUint64Value(value, p), names, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func Uint64Var(p *uint64, names []string, value uint64, usage string) { + CommandLine.Var(newUint64Value(value, p), names, usage) +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func (f *FlagSet) Uint64(names []string, value uint64, usage string) *uint64 { + p := new(uint64) + f.Uint64Var(p, names, value, usage) + return p +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func Uint64(names []string, value uint64, usage string) *uint64 { + return CommandLine.Uint64(names, value, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func (f *FlagSet) StringVar(p *string, names []string, value string, usage string) { + f.Var(newStringValue(value, p), names, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func StringVar(p *string, names []string, value string, usage string) { + CommandLine.Var(newStringValue(value, p), names, usage) +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func (f *FlagSet) String(names []string, value string, usage string) *string { + p := new(string) + f.StringVar(p, names, value, usage) + return p +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func String(names []string, value string, usage string) *string { + return CommandLine.String(names, value, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func (f *FlagSet) Float64Var(p *float64, names []string, value float64, usage string) { + f.Var(newFloat64Value(value, p), names, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func Float64Var(p *float64, names []string, value float64, usage string) { + CommandLine.Var(newFloat64Value(value, p), names, usage) +} + +// Float64 defines a float64 flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func (f *FlagSet) Float64(names []string, value float64, usage string) *float64 { + p := new(float64) + f.Float64Var(p, names, value, usage) + return p +} + +// Float64 defines a float64 flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func Float64(names []string, value float64, usage string) *float64 { + return CommandLine.Float64(names, value, usage) +} + +// DurationVar defines a time.Duration flag with specified name, default value, and usage string. +// The argument p points to a time.Duration variable in which to store the value of the flag. +func (f *FlagSet) DurationVar(p *time.Duration, names []string, value time.Duration, usage string) { + f.Var(newDurationValue(value, p), names, usage) +} + +// DurationVar defines a time.Duration flag with specified name, default value, and usage string. +// The argument p points to a time.Duration variable in which to store the value of the flag. +func DurationVar(p *time.Duration, names []string, value time.Duration, usage string) { + CommandLine.Var(newDurationValue(value, p), names, usage) +} + +// Duration defines a time.Duration flag with specified name, default value, and usage string. +// The return value is the address of a time.Duration variable that stores the value of the flag. +func (f *FlagSet) Duration(names []string, value time.Duration, usage string) *time.Duration { + p := new(time.Duration) + f.DurationVar(p, names, value, usage) + return p +} + +// Duration defines a time.Duration flag with specified name, default value, and usage string. +// The return value is the address of a time.Duration variable that stores the value of the flag. +func Duration(names []string, value time.Duration, usage string) *time.Duration { + return CommandLine.Duration(names, value, usage) +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func (f *FlagSet) Var(value Value, names []string, usage string) { + // Remember the default value as a string; it won't change. + flag := &Flag{names, usage, value, value.String()} + for _, name := range names { + name = strings.TrimPrefix(name, "#") + _, alreadythere := f.formal[name] + if alreadythere { + var msg string + if f.name == "" { + msg = fmt.Sprintf("flag redefined: %s", name) + } else { + msg = fmt.Sprintf("%s flag redefined: %s", f.name, name) + } + fmt.Fprintln(f.Out(), msg) + panic(msg) // Happens only if flags are declared with identical names + } + if f.formal == nil { + f.formal = make(map[string]*Flag) + } + f.formal[name] = flag + } +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func Var(value Value, names []string, usage string) { + CommandLine.Var(value, names, usage) +} + +// failf prints to standard error a formatted error and usage message and +// returns the error. +func (f *FlagSet) failf(format string, a ...interface{}) error { + err := fmt.Errorf(format, a...) + fmt.Fprintln(f.Out(), err) + if os.Args[0] == f.name { + fmt.Fprintf(f.Out(), "See '%s --help'.\n", os.Args[0]) + } else { + fmt.Fprintf(f.Out(), "See '%s %s --help'.\n", os.Args[0], f.name) + } + return err +} + +// usage calls the Usage method for the flag set, or the usage function if +// the flag set is CommandLine. +func (f *FlagSet) usage() { + if f == CommandLine { + Usage() + } else if f.Usage == nil { + defaultUsage(f) + } else { + f.Usage() + } +} + +func trimQuotes(str string) string { + if len(str) == 0 { + return str + } + type quote struct { + start, end byte + } + + // All valid quote types. + quotes := []quote{ + // Double quotes + { + start: '"', + end: '"', + }, + + // Single quotes + { + start: '\'', + end: '\'', + }, + } + + for _, quote := range quotes { + // Only strip if outermost match. + if str[0] == quote.start && str[len(str)-1] == quote.end { + str = str[1 : len(str)-1] + break + } + } + + return str +} + +// parseOne parses one flag. It reports whether a flag was seen. +func (f *FlagSet) parseOne() (bool, string, error) { + if len(f.args) == 0 { + return false, "", nil + } + s := f.args[0] + if len(s) == 0 || s[0] != '-' || len(s) == 1 { + return false, "", nil + } + if s[1] == '-' && len(s) == 2 { // "--" terminates the flags + f.args = f.args[1:] + return false, "", nil + } + name := s[1:] + if len(name) == 0 || name[0] == '=' { + return false, "", f.failf("bad flag syntax: %s", s) + } + + // it's a flag. does it have an argument? + f.args = f.args[1:] + hasValue := false + value := "" + if i := strings.Index(name, "="); i != -1 { + value = trimQuotes(name[i+1:]) + hasValue = true + name = name[:i] + } + + m := f.formal + flag, alreadythere := m[name] // BUG + if !alreadythere { + if name == "-help" || name == "help" || name == "h" { // special case for nice help message. + f.usage() + return false, "", ErrHelp + } + if len(name) > 0 && name[0] == '-' { + return false, "", f.failf("flag provided but not defined: -%s", name) + } + return false, name, ErrRetry + } + + if fv, ok := flag.Value.(boolFlag); ok && fv.IsBoolFlag() { // special case: doesn't need an arg + if hasValue { + if err := fv.Set(value); err != nil { + return false, "", f.failf("invalid boolean value %q for -%s: %v", value, name, err) + } + } else { + fv.Set("true") + } + } else { + // It must have a value, which might be the next argument. + if !hasValue && len(f.args) > 0 { + // value is the next arg + hasValue = true + value, f.args = f.args[0], f.args[1:] + } + if !hasValue { + return false, "", f.failf("flag needs an argument: -%s", name) + } + if err := flag.Value.Set(value); err != nil { + return false, "", f.failf("invalid value %q for flag -%s: %v", value, name, err) + } + } + if f.actual == nil { + f.actual = make(map[string]*Flag) + } + f.actual[name] = flag + for i, n := range flag.Names { + if n == fmt.Sprintf("#%s", name) { + replacement := "" + for j := i; j < len(flag.Names); j++ { + if flag.Names[j][0] != '#' { + replacement = flag.Names[j] + break + } + } + if replacement != "" { + fmt.Fprintf(f.Out(), "Warning: '-%s' is deprecated, it will be replaced by '-%s' soon. See usage.\n", name, replacement) + } else { + fmt.Fprintf(f.Out(), "Warning: '-%s' is deprecated, it will be removed soon. See usage.\n", name) + } + } + } + return true, "", nil +} + +// Parse parses flag definitions from the argument list, which should not +// include the command name. Must be called after all flags in the FlagSet +// are defined and before flags are accessed by the program. +// The return value will be ErrHelp if -help was set but not defined. +func (f *FlagSet) Parse(arguments []string) error { + f.parsed = true + f.args = arguments + for { + seen, name, err := f.parseOne() + + if seen { + continue + } + + if err == nil { + break + } + + if err == ErrRetry { + if len(name) > 1 { + err = nil + for _, letter := range strings.Split(name, "") { + f.args = append([]string{"-" + letter}, f.args...) + seen2, _, err2 := f.parseOne() + if seen2 { + continue + } + if err2 != nil { + err = f.failf("flag provided but not defined: -%s", name) + break + } + } + if err == nil { + continue + } + } else { + err = f.failf("flag provided but not defined: -%s", name) + } + } + + switch f.errorHandling { + case ContinueOnError: + return err + case ExitOnError: + os.Exit(2) + case PanicOnError: + panic(err) + } + + } + + return nil +} + +// ParseFlags is a utility function that adds a help flag if withHelp is true, +// calls cmd.Parse(args) and prints a relevant error message if there are +// incorrect number of arguments. It returns error only if error handling is +// set to ContinueOnError and parsing fails. If error handling is set to +// ExitOnError, it's safe to ignore the return value. +func (cmd *FlagSet) ParseFlags(args []string, withHelp bool) error { + var help *bool + if withHelp { + help = cmd.Bool([]string{"#help", "-help"}, false, "Print usage") + } + if err := cmd.Parse(args); err != nil { + return err + } + if help != nil && *help { + cmd.Usage() + // just in case Usage does not exit + os.Exit(0) + } + if str := cmd.CheckArgs(); str != "" { + cmd.ReportError(str, withHelp) + } + return nil +} + +func (cmd *FlagSet) ReportError(str string, withHelp bool) { + if withHelp { + if os.Args[0] == cmd.Name() { + str += ". See '" + os.Args[0] + " --help'" + } else { + str += ". See '" + os.Args[0] + " " + cmd.Name() + " --help'" + } + } + fmt.Fprintf(cmd.Out(), "docker: %s.\n", str) + os.Exit(1) +} + +// Parsed reports whether f.Parse has been called. +func (f *FlagSet) Parsed() bool { + return f.parsed +} + +// Parse parses the command-line flags from os.Args[1:]. Must be called +// after all flags are defined and before flags are accessed by the program. +func Parse() { + // Ignore errors; CommandLine is set for ExitOnError. + CommandLine.Parse(os.Args[1:]) +} + +// Parsed returns true if the command-line flags have been parsed. +func Parsed() bool { + return CommandLine.Parsed() +} + +// CommandLine is the default set of command-line flags, parsed from os.Args. +// The top-level functions such as BoolVar, Arg, and on are wrappers for the +// methods of CommandLine. +var CommandLine = NewFlagSet(os.Args[0], ExitOnError) + +// NewFlagSet returns a new, empty flag set with the specified name and +// error handling property. +func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet { + f := &FlagSet{ + name: name, + errorHandling: errorHandling, + } + return f +} + +// Init sets the name and error handling property for a flag set. +// By default, the zero FlagSet uses an empty name and the +// ContinueOnError error handling policy. +func (f *FlagSet) Init(name string, errorHandling ErrorHandling) { + f.name = name + f.errorHandling = errorHandling +}