diff --git a/README.md b/README.md index 5aaabea..d7188af 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ server and then offer them as file downloads with a unique URL. The option to de after a set time period discourages users from using the server as a permanent file store. #### Dependencies +* golang (https://golang.org/dl/ - for building as standalone) * yarn (https://yarnpkg.com/ - for the kiwiirc plugin UI) #### Downloading the file upload server's source code @@ -25,6 +26,7 @@ $ go run . ``` #### Building the server for production + ```console $ go build ``` @@ -75,7 +77,7 @@ Add the plugin javascript file to your kiwiirc `config.json` and configure the s If you're running the fileuploader server as a webircgateway plugin, use the webircgateway hostname, e.g. ```json - "server": "https://ws.irc.example.com/files", + "server": "https://ws.irc.example.com/files", ``` ## Database configuration diff --git a/expirer/expirer.go b/expirer/expirer.go index fe4d2ea..c163854 100644 --- a/expirer/expirer.go +++ b/expirer/expirer.go @@ -12,17 +12,19 @@ type Expirer struct { store *shardedfilestore.ShardedFileStore maxAge time.Duration identifiedMaxAge time.Duration + deletedMaxAge time.Duration jwtSecretsByIssuer map[string]string quitChan chan struct{} // closes when ticker has been stopped log *zerolog.Logger } -func New(store *shardedfilestore.ShardedFileStore, maxAge, identifiedMaxAge, checkInterval time.Duration, jwtSecretsByIssuer map[string]string, log *zerolog.Logger) *Expirer { +func New(store *shardedfilestore.ShardedFileStore, maxAge, identifiedMaxAge, deletedMaxAge, checkInterval time.Duration, jwtSecretsByIssuer map[string]string, log *zerolog.Logger) *Expirer { expirer := &Expirer{ ticker: time.NewTicker(checkInterval), store: store, maxAge: maxAge, identifiedMaxAge: identifiedMaxAge, + deletedMaxAge: deletedMaxAge, jwtSecretsByIssuer: jwtSecretsByIssuer, quitChan: make(chan struct{}), log: log, @@ -81,6 +83,13 @@ func (expirer *Expirer) gc(t time.Time) { Str("id", id). Msg("Terminated upload id") } + + err = expirer.deleteExpired() + if err != nil { + expirer.log.Error(). + Err(err). + Msg("Failed to purge deleted uploads from database") + } } func (expirer *Expirer) getExpired() (expiredIds []string, err error) { @@ -115,3 +124,38 @@ func (expirer *Expirer) getExpired() (expiredIds []string, err error) { return } + +func (expirer *Expirer) deleteExpired() (err error) { + switch expirer.store.DBConn.DBConfig.DriverName { + case "sqlite3": + _, err = expirer.store.DBConn.DB.Exec(` + DELETE FROM uploads + WHERE + CAST(strftime('%s', 'now') AS INTEGER) -- current time + >= + created_at + (CASE WHEN jwt_account IS NULL THEN $1 ELSE $2 END) + $3 -- expiration time + AND deleted == 1 + `, + expirer.maxAge.Seconds(), + expirer.identifiedMaxAge.Seconds(), + expirer.deletedMaxAge.Seconds(), + ) + case "mysql": + _, err = expirer.store.DBConn.DB.Exec(` + DELETE FROM uploads + WHERE + UNIX_TIMESTAMP() -- current time + >= + created_at + (CASE WHEN jwt_account IS NULL THEN ? ELSE ? END) + ? -- expiration time + AND deleted == 1 + `, + expirer.maxAge.Seconds(), + expirer.identifiedMaxAge.Seconds(), + expirer.deletedMaxAge.Seconds(), + ) + default: + panic("Unhandled database driver") + } + + return +} diff --git a/fallback-embed/fallback-embed-provider.go b/fallback-embed/fallback-embed-provider.go new file mode 100644 index 0000000..10ff07c --- /dev/null +++ b/fallback-embed/fallback-embed-provider.go @@ -0,0 +1,135 @@ +package fallbackembed + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// FallbackEmbed represents this package +type FallbackEmbed struct { + data *Data + httpClient *http.Client + targetKey string + providerURL string +} + +// Data represents the data for FallbackEmbed providers +type Data []struct { + Name string `json:"name"` + Patterns []Regex `json:"patterns"` +} + +// New returns a FallbackEmbed object +func New(providerURL, targetKey string) *FallbackEmbed { + obj := &FallbackEmbed{ + httpClient: &http.Client{ + Timeout: time.Second * 30, + }, + providerURL: providerURL, + targetKey: targetKey, + } + + return obj +} + +// ParseProviders parses the raw json obtained from noembed.com +func (f *FallbackEmbed) ParseProviders(buf io.Reader) error { + data, err := ioutil.ReadAll(buf) + if err != nil { + return err + } + + var providerData Data + err = json.Unmarshal(data, &providerData) + if err != nil { + return err + } + + f.data = &providerData + return nil +} + +// Get returns html string +func (f *FallbackEmbed) Get(wantedURL string, width int, height int) (html string, err error) { + if !f.ValidURL(wantedURL) { + return + } + + // Do replacements + reqURL := strings.Replace(f.providerURL, "{url}", url.QueryEscape(wantedURL), 1) + reqURL = strings.Replace(reqURL, "{width}", strconv.Itoa(width), 1) + reqURL = strings.Replace(reqURL, "{height}", strconv.Itoa(height), 1) + + var httpResp *http.Response + httpResp, err = f.httpClient.Get(reqURL) + if err != nil { + return + } + defer httpResp.Body.Close() + + var body []byte + body, err = ioutil.ReadAll(httpResp.Body) + if err != nil { + return + } + + // Try to parse json response + resp := make(map[string]interface{}) + err = json.Unmarshal(body, &resp) + if err != nil { + return + } + + // Check targetKey exists + if jsonVal, ok := resp[f.targetKey]; ok { + // Check targetVal is string + if htmlString, ok := jsonVal.(string); ok { + html = htmlString + return + } + } + + // Check for error key in json response + if jsonVal, ok := resp["error"]; ok { + // Check error is string + if errorString, ok := jsonVal.(string); ok { + err = errors.New(errorString) + return + } + } + + err = errors.New(`Fallback embed provider did not include a JSON property of "` + f.targetKey + `"`) + return +} + +// ValidURL is used to test if a url is supported by noembed +func (f *FallbackEmbed) ValidURL(url string) bool { + for _, entry := range *f.data { + for _, pattern := range entry.Patterns { + if pattern.Regexp.MatchString(url) { + return true + } + } + } + return false +} + +// Regex Unmarshaler +type Regex struct { + regexp.Regexp +} + +// UnmarshalText used to unmarshal regexp's from text +func (r *Regex) UnmarshalText(text []byte) error { + reg, err := regexp.Compile(string(text)) + r.Regexp = *reg + return err +} diff --git a/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue b/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue new file mode 100644 index 0000000..2485976 --- /dev/null +++ b/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue @@ -0,0 +1,146 @@ +