From 19cd29f827a4e8413b93d98d5ec1045d441070f2 Mon Sep 17 00:00:00 2001 From: Chad Arimura Date: Fri, 7 Dec 2018 17:38:46 -0800 Subject: [PATCH 1/2] remove receiver, add scraper, few mods --- functions/image-processor/func.py | 16 +- functions/image-processor/func.yaml | 5 +- functions/image-processor/func.yaml.bak | 28 -- functions/image-processor/payload.sample.json | 6 +- functions/receiver/Gopkg.lock | 18 -- functions/receiver/Gopkg.toml | 8 - functions/receiver/aws.go | 38 --- functions/receiver/cloudevent.go | 28 -- functions/receiver/func.go | 106 ------ functions/receiver/func.yaml | 14 - functions/receiver/payloads/aws.payload.json | 25 -- .../receiver/payloads/azure.payload.json | 24 -- functions/receiver/s3.go | 24 -- .../fnproject/fdk-go/CONTRIBUTING.md | 3 - .../github.com/fnproject/fdk-go/LICENSE | 202 ------------ .../github.com/fnproject/fdk-go/README.md | 83 ----- .../fnproject/fdk-go/examples/README.md | 59 ---- .../fdk-go/examples/hello/Gopkg.lock | 18 -- .../fdk-go/examples/hello/Gopkg.toml | 25 -- .../fnproject/fdk-go/examples/hello/func.go | 32 -- .../vendor/github.com/fnproject/fdk-go/fdk.go | 82 ----- .../github.com/fnproject/fdk-go/fdk_test.go | 305 ------------------ .../fnproject/fdk-go/utils/cloudevent.go | 108 ------- .../github.com/fnproject/fdk-go/utils/http.go | 62 ---- .../github.com/fnproject/fdk-go/utils/json.go | 96 ------ .../fnproject/fdk-go/utils/utils.go | 194 ----------- functions/scraper/Dockerfile | 16 + functions/scraper/func.py | 70 ++++ functions/scraper/func.yaml | 11 + functions/scraper/payload.json | 5 + functions/scraper/payload_faces.json | 7 + functions/scraper/requirements.txt | 3 + functions/word-generator/func.yaml | 2 +- .../fnproject/fdk-go/examples/hello/func.yaml | 2 +- setenv.sh | 3 + 35 files changed, 127 insertions(+), 1601 deletions(-) delete mode 100755 functions/image-processor/func.yaml.bak delete mode 100644 functions/receiver/Gopkg.lock delete mode 100644 functions/receiver/Gopkg.toml delete mode 100644 functions/receiver/aws.go delete mode 100644 functions/receiver/cloudevent.go delete mode 100644 functions/receiver/func.go delete mode 100644 functions/receiver/func.yaml delete mode 100644 functions/receiver/payloads/aws.payload.json delete mode 100644 functions/receiver/payloads/azure.payload.json delete mode 100644 functions/receiver/s3.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/CONTRIBUTING.md delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/LICENSE delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/README.md delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/examples/README.md delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.lock delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.toml delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/func.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/fdk.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/fdk_test.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/utils/cloudevent.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/utils/http.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/utils/json.go delete mode 100644 functions/receiver/vendor/github.com/fnproject/fdk-go/utils/utils.go create mode 100644 functions/scraper/Dockerfile create mode 100644 functions/scraper/func.py create mode 100644 functions/scraper/func.yaml create mode 100644 functions/scraper/payload.json create mode 100644 functions/scraper/payload_faces.json create mode 100644 functions/scraper/requirements.txt diff --git a/functions/image-processor/func.py b/functions/image-processor/func.py index b26f747..1f0db30 100644 --- a/functions/image-processor/func.py +++ b/functions/image-processor/func.py @@ -186,8 +186,7 @@ def setup_img_path(media_url): return '/tmp/poster_%s.jpeg' % h.hexdigest() -def post_media(slack_client, slack_channel, - filename, entities, status, log): +def post_media(slack_client, slack_channel, filename, entities, status, log): medias = entities.get("media", []) for media in medias: if "media_url_https" in media: @@ -197,8 +196,7 @@ def post_media(slack_client, slack_channel, slack_client, slack_channel, filename, url, status ) if "ok" in response and response["ok"]: - log.info("message posted to Slack successfully " - "from image: {0}".format(url)) + log.info("message posted to Slack successfully from image: {0}".format(url)) else: if "headers" in response: hs = response["headers"] @@ -213,8 +211,7 @@ def post_media(slack_client, slack_channel, raise Exception(ujson.dumps(response)) -def post_image_to_slack(slack_client, slack_channel, - filename, img_url, status): +def post_image_to_slack(slack_client, slack_channel,filename, img_url, status): return slack_client.api_call( "chat.postMessage", channel=slack_channel, @@ -301,8 +298,7 @@ def add_fn_logo(img): return cv.cvtColor(np.array(img_pil), cv.COLOR_RGB2BGR) -def process_single_media_file(ctx, sess, media_url, label_map, - event_id, event_type, ran_on): +def process_single_media_file(ctx, sess, media_url, label_map, event_id, event_type, ran_on): log = get_logger(ctx) scores = [] classes = [] @@ -310,8 +306,7 @@ def process_single_media_file(ctx, sess, media_url, label_map, num_detections = int(out[0][0]) log.info("detection completed, objects found: %s" % num_detections) for i in range(num_detections): - class_id, score, img = process_detection( - out, img, label_map, i, log) + class_id, score, img = process_detection(out, img, label_map, i, log) scores.append(score) classes.append(class_id) @@ -338,6 +333,7 @@ def with_graph(label_map): def fn(ctx, data=None, loop=None): log = get_logger(ctx) log.info("tf graph imported") + if data is not None or len(data) !=0: data = ujson.loads(data) log.info("incoming data: {0}".format(ujson.dumps(data))) diff --git a/functions/image-processor/func.yaml b/functions/image-processor/func.yaml index 9c97ef5..257ce45 100644 --- a/functions/image-processor/func.yaml +++ b/functions/image-processor/func.yaml @@ -1,6 +1,6 @@ schema_version: 20180708 name: image-processor -version: 0.0.101 +version: 0.0.104 runtime: docker format: http-stream memory: 1024 @@ -8,7 +8,6 @@ timeout: 300 idle_timeout: 300 config: DETECT_SENSITIVITY: "0.3" - IS_DOCKER4MAC_LOCAL: "true" expects: config: - name: DETECT_SENSITIVITY @@ -21,8 +20,6 @@ expects: required: true - name: TWITTER_ACCESS_TOKEN_SECRET required: true - - name: IS_DOCKER4MAC_LOCAL - required: false - name: SLACK_API_TOKEN required: true - name: SLACK_CHANNEL diff --git a/functions/image-processor/func.yaml.bak b/functions/image-processor/func.yaml.bak deleted file mode 100755 index 05cf495..0000000 --- a/functions/image-processor/func.yaml.bak +++ /dev/null @@ -1,28 +0,0 @@ -config: - DETECT_SENSITIVITY: "0.3" - IS_DOCKER4MAC_LOCAL: "true" -expects: - config: - - name: DETECT_SENSITIVITY - required: true - - name: TWITTER_CONSUMER_KEY - required: true - - name: TWITTER_CONSUMER_SECRET - required: true - - name: TWITTER_ACCESS_TOKEN_KEY - required: true - - name: TWITTER_ACCESS_TOKEN_SECRET - required: true - - name: IS_DOCKER4MAC_LOCAL - required: false - - name: SLACK_API_TOKEN - required: true - - name: SLACK_CHANNEL - required: true -format: json -memory: 1024 -name: image-processor -runtime: docker -timeout: 360 -type: async -version: 0.0.97 diff --git a/functions/image-processor/payload.sample.json b/functions/image-processor/payload.sample.json index 49d203f..47b70a3 100644 --- a/functions/image-processor/payload.sample.json +++ b/functions/image-processor/payload.sample.json @@ -1,8 +1,8 @@ { "media": [ - "http://survivingchurch.org/wp-content/uploads/2016/10/Donald-Trump-Photos-HD-1024x768.png", - "http://2wk128489wjq47m3kwxwe9hh.wpengine.netdna-cdn.com/wp-content/uploads/2014/10/hotdog_mustard-main.jpg", "https://i.ytimg.com/vi/Lu8lDgKInSM/hqdefault.jpg", + "http://survivingchurch.org/wp-content/uploads/2016/10/Donald-Trump-Photos-HD-1024x768.png", + "https://www.dairyqueen.com/Global/Food/Hot-Dogs_8-to-1_470x500.jpg", "https://pbs.twimg.com/profile_images/625802912887783424/_uzj-IZ4.jpg", "https://i.ytimg.com/vi/lFarE1hH0ss/maxresdefault.jpg", "http://m.step.aero/Burningman_2017_gifting_plane_rides.jpg", @@ -10,7 +10,7 @@ "http://m.step.aero/Chad's%20Dog.png" ], "media_test": [ - "http://survivingchurch.org/wp-content/uploads/2016/10/Donald-Trump-Photos-HD-1024x768.png" + "https://i.ytimg.com/vi/Lu8lDgKInSM/hqdefault.jpg" ], "event_id": "test_id", "event_type": "test_type", diff --git a/functions/receiver/Gopkg.lock b/functions/receiver/Gopkg.lock deleted file mode 100644 index 20e0bd0..0000000 --- a/functions/receiver/Gopkg.lock +++ /dev/null @@ -1,18 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/fnproject/fdk-go" - packages = [ - ".", - "utils" - ] - revision = "583f67630cc8fe3bcb5b251d2d99af4fe4911e09" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "c55f0d3da5ec2e9e5c9a7c563702e4cf28513fa1aaea1c18664ca2cb7d726f89" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/functions/receiver/Gopkg.toml b/functions/receiver/Gopkg.toml deleted file mode 100644 index c330ebe..0000000 --- a/functions/receiver/Gopkg.toml +++ /dev/null @@ -1,8 +0,0 @@ - -[[constraint]] - branch = "master" - name = "github.com/fnproject/fdk-go" - -[prune] - go-tests = true - unused-packages = true diff --git a/functions/receiver/aws.go b/functions/receiver/aws.go deleted file mode 100644 index 668f952..0000000 --- a/functions/receiver/aws.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" -) - -type AWSBucket struct { - Name string `json:"name"` -} - -type AWSObject struct { - Key string `json:"key"` -} - -type AWSData struct { - S3SchemaVersion string `json:"s3SchemaVersion"` - ConfigurationID string `json:"configurationId"` - Bucket AWSBucket `json:"bucket"` - Object AWSObject `json:"object"` -} - -func ParseAWSData(ceData interface{}) (*string, error) { - b, err := json.Marshal(ceData) - if err != nil { - return nil, err - } - - var d AWSData - err = json.Unmarshal(b, &d) - if err != nil { - return nil, err - } - - imgURL := fmt.Sprintf("https://s3.amazonaws.com/%s/%s", d.Bucket.Name, d.Object.Key) - - return &imgURL, nil -} diff --git a/functions/receiver/cloudevent.go b/functions/receiver/cloudevent.go deleted file mode 100644 index 40d721e..0000000 --- a/functions/receiver/cloudevent.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "strings" - "time" -) - -type CloudEvent struct { - CloudEventsVersion string `json:"cloudEventsVersion"` - EventID string `json:"eventID"` - Source string `json:"source"` - EventType string `json:"eventType"` - EventTypeVersion string `json:"eventTypeVersion"` - EventTime time.Time `json:"eventTime"` - SchemaURL string `json:"schemaURL"` - ContentType string `json:"contentType"` - Data interface{} `json:"data"` -} - -func GetImageURL(ce *CloudEvent) (*string, error) { - if strings.Contains(ce.EventType, "aws.s3.object.created") { - return ParseAWSData(ce.Data) - } - if strings.Contains(ce.EventType, "Microsoft.Storage.BlobCreated") { - return ParseAzureData(ce.Data) - } - return nil, nil -} diff --git a/functions/receiver/func.go b/functions/receiver/func.go deleted file mode 100644 index 9cc800f..0000000 --- a/functions/receiver/func.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "net/url" - "os" - - "github.com/fnproject/fdk-go" -) - -func main() { - fdk.Handle(fdk.HandlerFunc(withError)) -} - -func withError(ctx context.Context, in io.Reader, out io.Writer) { - err := myHandler(ctx, in) - if err != nil { - log.Println("unable to decode incoming stream, got error: ", err.Error()) - fdk.WriteStatus(out, http.StatusInternalServerError) - out.Write([]byte(err.Error())) - return - } -} - -type MediaProcessor struct { - EventID string `json:"event_id"` - EventType string `json:"event_type"` - MediaURL []string `json:"media"` -} - -func withDefault(key, defaultValue string) string { - envValue := os.Getenv(key) - if envValue == "" { - return defaultValue - } - return envValue -} - -func myHandler(ctx context.Context, in io.Reader) error { - var ce CloudEvent - err := json.NewDecoder(in).Decode(&ce) - if err != nil { - return err - } - - imgURL, err := GetImageURL(&ce) - if err != nil { - return err - } - - fctx := fdk.Context(ctx) - u, _ := url.Parse(fctx.RequestURL) - fnAPIURL := fctx.RequestURL[:len(fctx.RequestURL)-len(u.EscapedPath())] - fnAPIURL = withDefault("FN_API_URL", fnAPIURL) - - // if os.Getenv("IS_DOCKER4MAC_LOCAL") == "true" { - // fnAPIURL = "http://docker.for.mac.localhost:8080" - // } - - log.Println("Fn API URL: ", fnAPIURL) - - req, _ := http.NewRequest( - http.MethodPost, - fmt.Sprintf("%s/t/%s%s", fnAPIURL, os.Getenv("FN_APP_NAME"), "/image-processor"), - nil, - ) - - media := MediaProcessor{ - MediaURL: []string{ - *imgURL, - }, - EventType: ce.EventType, - EventID: ce.EventID, - } - var buf bytes.Buffer - err = json.NewEncoder(&buf).Encode(media) - if err != nil { - return err - } - - req.Body = ioutil.NopCloser(&buf) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - if resp.StatusCode > 202 { - return errors.New(string(b)) - } - - return nil -} diff --git a/functions/receiver/func.yaml b/functions/receiver/func.yaml deleted file mode 100644 index 021e9c4..0000000 --- a/functions/receiver/func.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: 0.0.43 -runtime: go -entrypoint: ./func -type: async -format: cloudevent -timeout: 360 -config: - DETECT_SENSITIVITY: "0.3" - FN_APP_NAME: cloudevents - IS_DOCKER4MAC_LOCAL: "true" -expects: - config: - - name: FN_API_URL - required: true diff --git a/functions/receiver/payloads/aws.payload.json b/functions/receiver/payloads/aws.payload.json deleted file mode 100644 index 06f585f..0000000 --- a/functions/receiver/payloads/aws.payload.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "eventType": "aws.s3.object.created", - "eventID": "C234-1234-1234", - "eventTime": "2018-04-26T14:48:09.769Z", - "cloudEventsVersion": "0.1", - "source": "https://serverless.com", - "extensions": {}, - "contentType": "application/json", - "eventTypeVersion": "2.0", - "data": { - "s3SchemaVersion": "1.0", - "configurationId": "cd267a38-30df-400e-9e3d-d0f1ca6e2410", - "bucket": { - "name": "cloudevents", - "ownerIdentity": {}, - "arn": "arn:aws:s3:::cloudevents" - }, - "object": { - "key": "dan_kohn.jpg", - "size": 444684, - "eTag": "38b01ff16138d7ca0a0eb3f7a88ff815", - "sequencer": "005AE1E6A9A3D61490" - } - } -} diff --git a/functions/receiver/payloads/azure.payload.json b/functions/receiver/payloads/azure.payload.json deleted file mode 100644 index 29b0c43..0000000 --- a/functions/receiver/payloads/azure.payload.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "eventType": "Microsoft.Storage.BlobCreated", - "eventID": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "eventTime": "2018-04-23T12:28:22.4579346Z", - "cloudEventsVersion": "0.1", - "source": "/subscriptions/326100e2-f69d-4268-8503-075374f62b6e/resourceGroups/cvtest34/providers/Microsoft.Storage/storageAccounts/cvtest34#/blobServices/default/containers/myfiles/blobs/IMG_20180224_0004.jpg", - "extensions": {}, - "contentType": "application/json", - "eventTypeVersion": "2.0", - "data": { - "api": "PutBlockList", - "clientRequestId": "a23b4aba-2755-4107-8020-8ba6c54b203d", - "requestId": "96fb5f0b-001e-0108-6dfe-da6e28000000", - "eTag": "0x8D5A915B425AFFD", - "contentType": "image/jpeg", - "contentLength": 2779325, - "blobType": "BlockBlob", - "url": "http://survivingchurch.org/wp-content/uploads/2016/10/Donald-Trump-Photos-HD-1024x768.png", - "sequencer": "000000000000000000000000000000BA00000000003db46c", - "storageDiagnostics": { - "batchId": "ba4fb664-f289-4742-8067-6c859411b066" - } - } -} diff --git a/functions/receiver/s3.go b/functions/receiver/s3.go deleted file mode 100644 index 4a07246..0000000 --- a/functions/receiver/s3.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "encoding/json" -) - -type AzureData struct { - URL string `json:"url"` -} - -func ParseAzureData(ceData interface{}) (*string, error) { - b, err := json.Marshal(ceData) - if err != nil { - return nil, err - } - - var d AzureData - err = json.Unmarshal(b, &d) - if err != nil { - return nil, err - } - - return &d.URL, nil -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/CONTRIBUTING.md b/functions/receiver/vendor/github.com/fnproject/fdk-go/CONTRIBUTING.md deleted file mode 100644 index f47058b..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/CONTRIBUTING.md +++ /dev/null @@ -1,3 +0,0 @@ -# Please refer to main contribution doc: - -https://github.com/fnproject/fn/blob/master/CONTRIBUTING.md diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/LICENSE b/functions/receiver/vendor/github.com/fnproject/fdk-go/LICENSE deleted file mode 100644 index 97ee2c3..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 Oracle Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/README.md b/functions/receiver/vendor/github.com/fnproject/fdk-go/README.md deleted file mode 100644 index 88c0c3c..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Go Fn Development Kit (FDK) - -[![GoDoc](https://godoc.org/github.com/fnproject/fdk-go?status.svg)](https://godoc.org/github.com/fnproject/fdk-go) - -fdk-go provides convenience functions for writing Go fn code - -For getting started with fn, please refer to https://github.com/fnproject/fn/blob/master/README.md - -# Installing fdk-go - -```sh -go get github.com/fnproject/fdk-go -``` - -or your favorite vendoring solution :) - -# Examples - -For a simple getting started, see the [examples](/examples/hello) and follow -the [README](/examples/README.md). If you already have `fn` set up it -will take 2 minutes! - -# Advanced example - -TODO going to move to [examples](examples/) too :) - -```go -package main - -import ( - "context" - "fmt" - "io" - "encoding/json" - - fdk "github.com/fnproject/fdk-go" - "net/http" -) - -func main() { - fdk.Handle(fdk.HandlerFunc(myHandler)) -} - -func myHandler(ctx context.Context, in io.Reader, out io.Writer) { - fnctx := fdk.Context(ctx) - - contentType := fnctx.Header.Get("Content-Type") - if contentType != "application/json" { - fdk.WriteStatus(out, 400) - fdk.SetHeader(out, "Content-Type", "application/json") - io.WriteString(out, `{"error":"invalid content type"}`) - return - } - - if fnctx.Config["FN_METHOD"] != "PUT" { - fdk.WriteStatus(out, 404) - fdk.SetHeader(out, "Content-Type", "application/json") - io.WriteString(out, `{"error":"route not found"}`) - return - } - - var person struct { - Name string `json:"name"` - } - json.NewDecoder(in).Decode(&person) - - // you can write your own headers & status, if you'd like to - fdk.WriteStatus(out, 201) - fdk.SetHeader(out, "Content-Type", "application/json") - - all := struct { - Name string `json:"name"` - Header http.Header `json:"header"` - Config map[string]string `json:"config"` - }{ - Name: person.Name, - Header: fnctx.Header, - Config: fnctx.Config, - } - - json.NewEncoder(out).Encode(&all) -} -``` diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/README.md b/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/README.md deleted file mode 100644 index 6c3e6dc..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Function Examples - -The goal of the `fdk`'s are to make it just as easy to write a hot function as -it is a cold one. The best way to showcase this is with an example. - -This is an example of a hot function using the fdk-go bindings. The [hot function -documentation](https://github.com/fnproject/fn/blob/master/docs/developers/hot-functions.md) -contains an analysis of how this example works under the hood. With any of the -examples provided here, you may use any format to configure your functions in -`fn` itself. Here we add instructions to set up functions with a 'hot' format. - -### How to run the example - -Install the CLI tool, start a Fn server and run `docker login` to login to -DockerHub. See the [front page](https://github.com/fnproject/fn) for -instructions. - -Initialize the example with an image name you can access: - -```sh -fn init --runtime docker --format http --name -``` - -`--format json` will also work here, or if your functions already use json -then adding the fdk will be seamless. - -Build and deploy the function to the Fn server (default localhost:8080) - -```sh -fn deploy --app hot-app -``` - -Now call your function (may take a sec to pull image): - -```sh -curl -X POST -d '{"name":"Clarice"}' http://localhost:8080/r/hot-app/hello -``` - -**Note** that this expects you were in a directory named 'hello' (where this -example lives), if this doesn't work, replace 'hello' with your `$PWD` from -the `deploy` command. - -Then call it again to see how fast hot functions are! - -### Details - -If you poke around in the Dockerfile you'll see that we're simply adding the -file found in this directory, getting the `fdk-go` package to our workspace -and then building a binary and building an image with that binary. That then -gets deployed to dockerhub and fn. - -For more robust projects, it's recommended to use a tool like `dep` or -`glide` to get dependencies such as the `fdk-go` into your functions. - -Scoping out `func.go` you can see that the handler code only deals with input -and output, and doesn't have to deal with decoding the formatting from -functions (i.e. i/o is presented through `io.Writer` and `io.Reader`). This -makes it much easier to write hot functions. - diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.lock b/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.lock deleted file mode 100644 index 1e5cae9..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.lock +++ /dev/null @@ -1,18 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/fnproject/fdk-go" - packages = [ - ".", - "utils" - ] - revision = "5d768b2006f11737b6a69a758ddd6d2fac04923e" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "c55f0d3da5ec2e9e5c9a7c563702e4cf28513fa1aaea1c18664ca2cb7d726f89" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.toml b/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.toml deleted file mode 100644 index 4af110c..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/Gopkg.toml +++ /dev/null @@ -1,25 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - branch = "master" - name = "github.com/fnproject/fdk-go" diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/func.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/func.go deleted file mode 100644 index 93f22c8..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/examples/hello/func.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - - fdk "github.com/fnproject/fdk-go" -) - -func main() { - fdk.Handle(fdk.HandlerFunc(myHandler)) -} - -func myHandler(ctx context.Context, in io.Reader, out io.Writer) { - var person struct { - Name string `json:"name"` - } - json.NewDecoder(in).Decode(&person) - if person.Name == "" { - person.Name = "World" - } - - msg := struct { - Msg string `json:"message"` - }{ - Msg: fmt.Sprintf("Hello %s", person.Name), - } - - json.NewEncoder(out).Encode(&msg) -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk.go deleted file mode 100644 index 42e0ed8..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk.go +++ /dev/null @@ -1,82 +0,0 @@ -package fdk - -import ( - "context" - "io" - "net/http" - "os" - - "github.com/fnproject/fdk-go/utils" -) - -type Handler interface { - Serve(ctx context.Context, in io.Reader, out io.Writer) -} - -type HandlerFunc func(ctx context.Context, in io.Reader, out io.Writer) - -func (f HandlerFunc) Serve(ctx context.Context, in io.Reader, out io.Writer) { - f(ctx, in, out) -} - -// Context will return an *fn.Ctx that can be used to read configuration and -// request information from an incoming request. -func Context(ctx context.Context) *Ctx { - utilsCtx := utils.Context(ctx) - return &Ctx{ - Header: utilsCtx.Header, - Config: utilsCtx.Config, - RequestURL: utilsCtx.RequestURL, - Method: utilsCtx.Method, - } -} - -func WithContext(ctx context.Context, fnctx *Ctx) context.Context { - utilsCtx := &utils.Ctx{ - Header: fnctx.Header, - Config: fnctx.Config, - RequestURL: fnctx.RequestURL, - Method: fnctx.Method, - } - return utils.WithContext(ctx, utilsCtx) -} - -// Ctx provides access to Config and Headers from fn. -type Ctx struct { - Header http.Header - Config map[string]string - RequestURL string - Method string -} - -// AddHeader will add a header on the function response, for hot function -// formats. -func AddHeader(out io.Writer, key, value string) { - if resp, ok := out.(*utils.Response); ok { - resp.Header.Add(key, value) - } -} - -// SetHeader will set a header on the function response, for hot function -// formats. -func SetHeader(out io.Writer, key, value string) { - if resp, ok := out.(*utils.Response); ok { - resp.Header.Set(key, value) - } -} - -// WriteStatus will set the status code to return in the function response, for -// hot function formats. -func WriteStatus(out io.Writer, status int) { - if resp, ok := out.(*utils.Response); ok { - resp.Status = status - } -} - -// Handle will run the event loop for a function. Handle should be invoked -// through main() in a user's function and can handle communication between the -// function and fn server via any of the supported formats. -func Handle(handler Handler) { - format, _ := os.LookupEnv("FN_FORMAT") - utils.Do(handler, format, os.Stdin, os.Stdout) -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk_test.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk_test.go deleted file mode 100644 index b3a0a88..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/fdk_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package fdk - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "os" - "strings" - "testing" - - "github.com/fnproject/fdk-go/utils" - "time" -) - -func echoHTTPHandler(_ context.Context, in io.Reader, out io.Writer) { - io.Copy(out, in) - WriteStatus(out, http.StatusTeapot+2) - SetHeader(out, "yo", "dawg") -} - -func TestHandler(t *testing.T) { - inString := "yodawg" - var in bytes.Buffer - io.WriteString(&in, inString) - - var out bytes.Buffer - echoHTTPHandler(utils.BuildCtx(), &in, &out) - - if out.String() != inString { - t.Fatalf("this was supposed to be easy. strings no matchy: %s got: %s", inString, out.String()) - } -} - -func TestDefault(t *testing.T) { - inString := "yodawg" - var in bytes.Buffer - io.WriteString(&in, inString) - - var out bytes.Buffer - - utils.DoDefault(HandlerFunc(echoHTTPHandler), utils.BuildCtx(), &in, &out) - - if out.String() != inString { - t.Fatalf("strings no matchy: %s got: %s", inString, out.String()) - } -} - -func JSONHandler(_ context.Context, in io.Reader, out io.Writer) { - var person struct { - Name string `json:"name"` - } - err := json.NewDecoder(in).Decode(&person) - if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - } else { - if person.Name == "" { - person.Name = "world" - } - - body := fmt.Sprintf("Hello %s!\n", person.Name) - SetHeader(out, "Content-Type", "application/json") - err = json.NewEncoder(out).Encode(body) - if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - } - } -} - -func JSONWithStatusCode(_ context.Context, _ io.Reader, out io.Writer) { - SetHeader(out, "Content-Type", "application/json") - WriteStatus(out, 201) -} - -func TestJSON(t *testing.T) { - req := &utils.JsonIn{ - CallID: "someid", - Body: `{"name":"john"}`, - ContentType: "application/json", - Deadline: "2018-01-30T16:52:39.786Z", - Protocol: utils.CallRequestHTTP{ - Type: "http", - RequestURL: "http://localhost:8080/r/myapp/yodawg", - Headers: http.Header{}, - Method: "POST", - }, - } - - var in bytes.Buffer - err := json.NewEncoder(&in).Encode(req) - if err != nil { - t.Fatal("Unable to marshal request") - } - - var out, buf bytes.Buffer - - err = utils.DoJSONOnce(HandlerFunc(JSONHandler), utils.BuildCtx(), &in, &out, &buf, make(http.Header)) - if err != nil { - t.Fatal("should not return error", err) - } - - JSONOut := &utils.JsonOut{} - err = json.NewDecoder(&out).Decode(JSONOut) - - if err != nil { - t.Fatal(err.Error()) - } - if !strings.Contains(JSONOut.Body, "Hello john!") { - t.Fatalf("Output assertion mismatch. Expected: `Hello john!\n`. Actual: %v", JSONOut.Body) - } - if JSONOut.Protocol.StatusCode != 200 { - t.Fatalf("Response code must equal to 200, but have: %v", JSONOut.Protocol.StatusCode) - } -} - -func TestFailedJSON(t *testing.T) { - dummyBody := "should fail with this" - in := strings.NewReader(dummyBody) - - var out, buf bytes.Buffer - - JSONOut := &utils.JsonOut{} - err := utils.DoJSONOnce(HandlerFunc(JSONHandler), utils.BuildCtx(), in, &out, &buf, make(http.Header)) - if err != nil { - t.Fatal("should not return error", err) - } - - err = json.NewDecoder(&out).Decode(JSONOut) - if err != nil { - t.Fatal(err.Error()) - } - if JSONOut.Protocol.StatusCode != 500 { - t.Fatalf("Response code must equal to 500, but have: %v", JSONOut.Protocol.StatusCode) - } -} - -func TestJSONEOF(t *testing.T) { - var in, out, buf bytes.Buffer - - err := utils.DoJSONOnce(HandlerFunc(JSONHandler), utils.BuildCtx(), &in, &out, &buf, make(http.Header)) - if err != io.EOF { - t.Fatal("should return EOF") - } -} - -func TestJSONOverwriteStatusCodeAndHeaders(t *testing.T) { - var out, buf bytes.Buffer - req := &utils.JsonIn{ - CallID: "someid", - Body: `{"name":"john"}`, - ContentType: "application/json", - Deadline: "2018-01-30T16:52:39.786Z", - Protocol: utils.CallRequestHTTP{ - Type: "json", - RequestURL: "http://localhost:8080/r/myapp/yodawg", - Headers: http.Header{}, - Method: "POST", - }, - } - - var in bytes.Buffer - err := json.NewEncoder(&in).Encode(req) - if err != nil { - t.Fatal("Unable to marshal request") - } - - err = utils.DoJSONOnce(HandlerFunc(JSONWithStatusCode), utils.BuildCtx(), &in, &out, &buf, make(http.Header)) - if err != nil { - t.Fatal("should not return error", err) - } - - JSONOut := &utils.JsonOut{} - err = json.NewDecoder(&out).Decode(JSONOut) - if err != nil { - t.Fatal(err.Error()) - } - - if JSONOut.Protocol.StatusCode != 201 { - t.Fatalf("Response code must equal to 201, but have: %v", JSONOut.Protocol.StatusCode) - } - cType := JSONOut.Protocol.Headers.Get("Content-Type") - if !strings.Contains(cType, "application/json") { - t.Fatalf("Response content type should be application/json in this test, but have: %v", cType) - } -} - -func TestHTTP(t *testing.T) { - // simulate fn writing us http requests... - - bodyString := "yodawg" - in := HTTPreq(t, bodyString) - - var out bytes.Buffer - ctx := utils.BuildCtx() - err := utils.DoHTTPOnce(HandlerFunc(echoHTTPHandler), ctx, in, &out, &bytes.Buffer{}, make(http.Header)) - if err != nil { - t.Fatal("should not return error", err) - } - - res, err := http.ReadResponse(bufio.NewReader(&out), nil) - if err != nil { - t.Fatal(err) - } - - if res.StatusCode != http.StatusTeapot+2 { - t.Fatal("got wrong status code", res.StatusCode) - } - - outBody, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatal(err) - } - - if res.Header.Get("yo") != "dawg" { - t.Fatal("expected yo dawg header, didn't get it") - } - - if string(outBody) != bodyString { - t.Fatal("strings no matchy for http", string(outBody), bodyString) - } - -} - -func TestHTTPEOF(t *testing.T) { - var in bytes.Buffer - var out bytes.Buffer - ctx := utils.BuildCtx() - - err := utils.DoHTTPOnce(HandlerFunc(echoHTTPHandler), ctx, &in, &out, &bytes.Buffer{}, make(http.Header)) - if err != io.EOF { - t.Fatal("should return EOF") - } -} - -func HTTPreq(t *testing.T, bod string) io.Reader { - req, err := http.NewRequest("GET", "http://localhost:8080/r/myapp/yodawg", strings.NewReader(bod)) - if err != nil { - t.Fatal(err) - } - - byts, err := httputil.DumpRequestOut(req, true) - if err != nil { - t.Fatal(err) - } - return bytes.NewReader(byts) -} - -func TestCloudEvent(t *testing.T) { - req := &utils.CloudEventIn{ - CloudEvent: utils.CloudEvent{ - EventID: "someid", - Source: "fn-api", - EventType: "test-type", - EventTypeVersion: "1.0", - EventTime: time.Now(), - ContentType: "application/json", - Data: `{"name": "John"}`, - }, - Extensions: utils.CloudEventInExtension{ - Deadline: "2018-01-30T16:52:39.786Z", - Protocol: utils.CallRequestHTTP{ - Type: "http", - RequestURL: "http://localhost:8080/r/myapp/yodawg", - Headers: http.Header{}, - Method: "POST", - }, - }, - } - - var in bytes.Buffer - err := json.NewEncoder(&in).Encode(req) - if err != nil { - t.Fatal("Unable to marshal request") - } - t.Log(in.String()) - var out, buf bytes.Buffer - - err = utils.DoCloudEventOnce(HandlerFunc(JSONHandler), utils.BuildCtx(), &in, &out, &buf, make(http.Header)) - if err != nil { - t.Fatal("should not return error", err) - } - - t.Log(out.String()) - ceOut := &utils.CloudEventOut{} - err = json.NewDecoder(&out).Decode(ceOut) - if err != nil { - t.Fatal(err.Error()) - } - - if ceOut.Extensions.Protocol.StatusCode != 200 { - t.Fatalf("Response code must equal to 200, but have: %v", ceOut.Extensions.Protocol.StatusCode) - } - - var respData string - json.Unmarshal([]byte(ceOut.Data.(string)), &respData) - - if !strings.Contains(respData, "Hello John!\n") { - t.Fatalf("Output assertion mismatch. Expected: `Hello John!\n`. Actual: %v", ceOut.Data) - } -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/cloudevent.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/cloudevent.go deleted file mode 100644 index 3aff99c..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/cloudevent.go +++ /dev/null @@ -1,108 +0,0 @@ -package utils - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "strings" - "time" -) - -type CloudEvent struct { - CloudEventsVersion string `json:"cloudEventsVersion"` - EventID string `json:"eventID"` - Source string `json:"source"` - EventType string `json:"eventType"` - EventTypeVersion string `json:"eventTypeVersion"` - EventTime time.Time `json:"eventTime"` - SchemaURL string `json:"schemaURL"` - ContentType string `json:"contentType"` - Data interface{} `json:"data"` // from docs: the payload is encoded into a media format which is specified by the contentType attribute (e.g. application/json) -} - -type CloudEventInExtension struct { - Protocol CallRequestHTTP `json:"protocol"` - Deadline string `json:"deadline"` -} - -type CloudEventOutExtension struct { - Protocol CallResponseHTTP `json:"protocol"` -} - -type CloudEventIn struct { - CloudEvent - Extensions CloudEventInExtension `json:"extensions"` -} - -type CloudEventOut struct { - CloudEvent - Extensions CloudEventOutExtension `json:"extensions"` -} - -func writeError(ceOut *CloudEventOut, err error) { - ceOut.Extensions.Protocol.StatusCode = http.StatusInternalServerError - ceOut.Data = fmt.Sprintf(`{"error": %v}`, err.Error()) - ceOut.ContentType = "application/json" - ceOut.EventTime = time.Now() -} - -func DoCloudEventOnce(handler Handler, ctx context.Context, in io.Reader, out io.Writer, buf *bytes.Buffer, hdr http.Header) error { - buf.Reset() - ResetHeaders(hdr) - resp := Response{ - Writer: buf, - Status: 200, - Header: hdr, - } - - ceOut := CloudEventOut{ - Extensions: CloudEventOutExtension{ - Protocol: CallResponseHTTP{ - StatusCode: http.StatusOK, - Headers: hdr, - }, - }, - CloudEvent: CloudEvent{ - ContentType: "text/plain", - }, - } - - var ceIn CloudEventIn - err := json.NewDecoder(in).Decode(&ceIn) - if err != nil { - if err == io.EOF { - return err - } - writeError(&ceOut, err) - } else { - SetHeaders(ctx, ceIn.Extensions.Protocol.Headers) - SetRequestURL(ctx, ceIn.Extensions.Protocol.RequestURL) - SetMethod(ctx, ceIn.Extensions.Protocol.Method) - ctx, cancel := CtxWithDeadline(ctx, ceIn.Extensions.Deadline) - defer cancel() - - handler.Serve(ctx, strings.NewReader(ceIn.Data.(string)), &resp) - } - ceOut.EventID = ceIn.EventID - ceOut.EventTime = time.Now() - ceOut.ContentType = ceOut.Extensions.Protocol.Headers.Get("Content-Type") - ceOut.Data = buf.String() - return json.NewEncoder(out).Encode(ceOut) -} - -func DoCloudEvent(handler Handler, ctx context.Context, in io.Reader, out io.Writer) { - var buf bytes.Buffer - hdr := make(http.Header) - - for { - err := DoCloudEventOnce(handler, ctx, in, out, &buf, hdr) - if err != nil { - log.Println(err.Error()) - break - } - } -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/http.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/http.go deleted file mode 100644 index e1cd3ce..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/http.go +++ /dev/null @@ -1,62 +0,0 @@ -package utils - -import ( - "bufio" - "bytes" - "context" - "io" - "io/ioutil" - "net/http" - "strconv" -) - -func GetHTTPResp(buf *bytes.Buffer, fnResp *Response, req *http.Request) http.Response { - - fnResp.Header.Set("Content-Length", strconv.Itoa(buf.Len())) - - hResp := http.Response{ - ProtoMajor: 1, - ProtoMinor: 1, - StatusCode: fnResp.Status, - Request: req, - Body: ioutil.NopCloser(buf), - ContentLength: int64(buf.Len()), - Header: fnResp.Header, - } - - return hResp -} - -func DoHTTPOnce(handler Handler, ctx context.Context, in io.Reader, out io.Writer, buf *bytes.Buffer, hdr http.Header) error { - buf.Reset() - ResetHeaders(hdr) - resp := Response{ - Writer: buf, - Status: 200, - Header: hdr, - } - - req, err := http.ReadRequest(bufio.NewReader(in)) - if err != nil { - // stdin now closed - if err == io.EOF { - return err - } - // TODO it would be nice if we could let the user format this response to their preferred style.. - resp.Status = http.StatusInternalServerError - io.WriteString(resp, err.Error()) - } else { - fnDeadline := Context(ctx).Header.Get("FN_DEADLINE") - ctx, cancel := CtxWithDeadline(ctx, fnDeadline) - defer cancel() - - SetHeaders(ctx, req.Header) - SetRequestURL(ctx, req.URL.String()) - SetMethod(ctx, req.Method) - handler.Serve(ctx, req.Body, &resp) - } - - hResp := GetHTTPResp(buf, &resp, req) - hResp.Write(out) - return nil -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/json.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/json.go deleted file mode 100644 index 68002cc..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/json.go +++ /dev/null @@ -1,96 +0,0 @@ -package utils - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" -) - -func DoJSON(handler Handler, ctx context.Context, in io.Reader, out io.Writer) { - var buf bytes.Buffer - hdr := make(http.Header) - - for { - err := DoJSONOnce(handler, ctx, in, out, &buf, hdr) - if err != nil { - break - } - } -} - -type CallRequestHTTP struct { - Type string `json:"type"` - RequestURL string `json:"request_url"` - Method string `json:"method"` - Headers http.Header `json:"headers"` -} - -type JsonIn struct { - CallID string `json:"call_id"` - Deadline string `json:"deadline"` - Body string `json:"body"` - ContentType string `json:"content_type"` - Protocol CallRequestHTTP `json:"protocol"` -} - -type CallResponseHTTP struct { - StatusCode int `json:"status_code,omitempty"` - Headers http.Header `json:"headers,omitempty"` -} - -type JsonOut struct { - Body string `json:"body"` - ContentType string `json:"content_type"` - Protocol CallResponseHTTP `json:"protocol,omitempty"` -} - -func GetJSONResp(buf *bytes.Buffer, fnResp *Response) *JsonOut { - - hResp := &JsonOut{ - Body: buf.String(), - ContentType: "", - Protocol: CallResponseHTTP{ - StatusCode: fnResp.Status, - Headers: fnResp.Header, - }, - } - - return hResp -} - -func DoJSONOnce(handler Handler, ctx context.Context, in io.Reader, out io.Writer, buf *bytes.Buffer, hdr http.Header) error { - buf.Reset() - ResetHeaders(hdr) - resp := Response{ - Writer: buf, - Status: 200, - Header: hdr, - } - - var jsonRequest JsonIn - err := json.NewDecoder(in).Decode(&jsonRequest) - if err != nil { - // stdin now closed - if err == io.EOF { - return err - } - resp.Status = http.StatusInternalServerError - io.WriteString(resp, fmt.Sprintf(`{"error": %v}`, err.Error())) - } else { - - SetHeaders(ctx, jsonRequest.Protocol.Headers) - SetRequestURL(ctx, jsonRequest.Protocol.RequestURL) - SetMethod(ctx, jsonRequest.Protocol.Method) - ctx, cancel := CtxWithDeadline(ctx, jsonRequest.Deadline) - defer cancel() - handler.Serve(ctx, strings.NewReader(jsonRequest.Body), &resp) - } - - jsonResponse := GetJSONResp(buf, &resp) - json.NewEncoder(out).Encode(jsonResponse) - return nil -} diff --git a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/utils.go b/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/utils.go deleted file mode 100644 index b735803..0000000 --- a/functions/receiver/vendor/github.com/fnproject/fdk-go/utils/utils.go +++ /dev/null @@ -1,194 +0,0 @@ -package utils - -import ( - "bytes" - "context" - "io" - "net/http" - "os" - "strings" - "time" -) - -type Handler interface { - Serve(ctx context.Context, in io.Reader, out io.Writer) -} - -// Context will return an *fn.Ctx that can be used to read configuration and -// request information from an incoming request. -func Context(ctx context.Context) *Ctx { - return ctx.Value(ctxKey).(*Ctx) -} - -func WithContext(ctx context.Context, fnctx *Ctx) context.Context { - return context.WithValue(ctx, ctxKey, fnctx) -} - -// Ctx provides access to Config and Headers from fn. -type Ctx struct { - Header http.Header - Config map[string]string - RequestURL string - Method string -} - -type key struct{} - -var ctxKey = new(key) - -func Do(handler Handler, format string, in io.Reader, out io.Writer) { - ctx := BuildCtx() - switch format { - case "http": - DoHTTP(handler, ctx, in, out) - case "json": - DoJSON(handler, ctx, in, out) - case "cloudevent": - DoCloudEvent(handler, ctx, in, out) - case "default": - DoDefault(handler, ctx, in, out) - default: - panic("unknown format (fdk-go): " + format) - } -} - -// doDefault only runs once, since it is a 'cold' function -func DoDefault(handler Handler, ctx context.Context, in io.Reader, out io.Writer) { - SetHeaders(ctx, BuildHeadersFromEnv()) - fnDeadline, _ := os.LookupEnv("FN_DEADLINE") - - ctx, cancel := CtxWithDeadline(ctx, fnDeadline) - defer cancel() - - handler.Serve(ctx, in, out) -} - -// doHTTP runs a loop, reading http requests from in and writing -// http responses to out -func DoHTTP(handler Handler, ctx context.Context, in io.Reader, out io.Writer) { - var buf bytes.Buffer - // maps don't get down-sized, so we can reuse this as it's likely that the - // user sends in the same amount of headers over and over (but still clear - // b/w runs) -- buf uses same principle - hdr := make(http.Header) - - for { - err := DoHTTPOnce(handler, ctx, in, out, &buf, hdr) - if err != nil { - break - } - } -} - -func CtxWithDeadline(ctx context.Context, fnDeadline string) (context.Context, context.CancelFunc) { - t, err := time.Parse(time.RFC3339, fnDeadline) - if err == nil { - return context.WithDeadline(ctx, t) - } - return context.WithCancel(ctx) -} - -func ResetHeaders(m http.Header) { - for k := range m { // compiler optimizes this to 1 instruction now - delete(m, k) - } -} - -// response is a general purpose response struct any format can use to record -// user's code responses before formatting them appropriately. -type Response struct { - Status int - Header http.Header - - io.Writer -} - -var ( - BaseHeaders = map[string]struct{}{ - `FN_APP_NAME`: struct{}{}, - `FN_PATH`: struct{}{}, - `FN_METHOD`: struct{}{}, - `FN_FORMAT`: struct{}{}, - `FN_MEMORY`: struct{}{}, - `FN_TYPE`: struct{}{}, - } - - HeaderPrefix = `FN_HEADER_` - - ExactHeaders = map[string]struct{}{ - `FN_CALL_ID`: struct{}{}, - `FN_METHOD`: struct{}{}, - `FN_REQUEST_URL`: struct{}{}, - } -) - -func SetHeaders(ctx context.Context, hdr http.Header) { - fctx := ctx.Value(ctxKey).(*Ctx) - fctx.Header = hdr -} - -func SetRequestURL(ctx context.Context, requestURL string) { - fctx := ctx.Value(ctxKey).(*Ctx) - fctx.RequestURL = requestURL -} - -func SetMethod(ctx context.Context, method string) { - fctx := ctx.Value(ctxKey).(*Ctx) - fctx.Method = method -} - -func BuildCtx() context.Context { - ctx := &Ctx{ - Config: BuildConfig(), - // allow caller to build headers separately (to avoid map alloc) - } - - return WithContext(context.Background(), ctx) -} - -func BuildConfig() map[string]string { - cfg := make(map[string]string, len(BaseHeaders)) - - for _, e := range os.Environ() { - vs := strings.SplitN(e, "=", 2) - if len(vs) < 2 { - vs = append(vs, "") - } - cfg[vs[0]] = vs[1] - } - return cfg -} - -func BuildHeadersFromEnv() http.Header { - env := os.Environ() - hdr := make(http.Header, len(env)-len(BaseHeaders)) - - for _, e := range env { - vs := strings.SplitN(e, "=", 2) - hdrKey := IsHeaderKey(vs[0]) - if hdrKey == "" { - continue - } - if len(vs) < 2 { - vs = append(vs, "") - } - // rebuild these as 'http' headers - vs = strings.Split(vs[1], ", ") - for _, v := range vs { - hdr.Add(hdrKey, v) - } - } - return hdr -} - -// for getting headers out of env -func IsHeaderKey(key string) string { - if strings.HasPrefix(key, HeaderPrefix) { - return strings.TrimPrefix(key, HeaderPrefix) - } - _, ok := ExactHeaders[key] - if ok { - return key - } - return "" -} diff --git a/functions/scraper/Dockerfile b/functions/scraper/Dockerfile new file mode 100644 index 0000000..306dfba --- /dev/null +++ b/functions/scraper/Dockerfile @@ -0,0 +1,16 @@ +FROM fnproject/python:3.7-dev as build-stage + +ARG FLICKR_API_KEY +ARG FLICKR_API_SECRET + +WORKDIR /function +ADD . /function/ + +RUN pip3 install --target /python/ --no-cache --no-cache-dir -r requirements.txt + +FROM fnproject/python:3.7 +WORKDIR /function +COPY --from=build-stage /function /function +COPY --from=build-stage /python /python +ENV PYTHONPATH=/python +ENTRYPOINT ["python3", "func.py"] diff --git a/functions/scraper/func.py b/functions/scraper/func.py new file mode 100644 index 0000000..f6a5737 --- /dev/null +++ b/functions/scraper/func.py @@ -0,0 +1,70 @@ +import fdk +import ujson +import os +import random +import flickrapi +import ssl +import sys +import requests +from fdk import fixtures + +ssl._create_default_https_context = ssl._create_unverified_context + + +flickr = flickrapi.FlickrAPI( + os.environ.get("FLICKR_API_KEY"), + os.environ.get("FLICKR_API_SECRET"), + token_cache_location='/tmp', + format='parsed-json' +) + +PHOTO_SOURCE_URL = 'https://farm{0}.staticflickr.com/{1}/{2}_{3}{4}.{5}' + +def get_image_url(photo_dict): + return PHOTO_SOURCE_URL.format( + photo_dict['farm'], photo_dict['server'], + photo_dict['id'], photo_dict['secret'], + '_c', 'jpg' + ) + + +def photo_to_payload(body, photo_dict): + return { + "id": photo_dict.get('id'), + "image_url": get_image_url(photo_dict), + "countrycode": body.get("countrycode"), + "bucket": body.get("bucket", "") + } + + +def handler(ctx, data=None, loop=None): + payloads = [] + if data and len(data) > 0: + body = ujson.loads(data) + photos = flickr.photos.search( + text=body.get("query", "baby smile"), + per_page=int(body.get("num", "5")), + page=int(body.get("page", int(random.uniform(1, 50)))), + extras="original_format", + safe_search="1", + content_type="1", + ) + + for p in photos.get('photos', {'photo': {}}).get('photo', []): + payloads.append(photo_to_payload(body, p)) + + for p in payloads: + this_payload = { + "media":[p.get("image_url")], + "event_id": "test_id", + "event_type": "test_type", + "ran_on": "" + } + process_url = "http://docker.for.mac.localhost:8080/t/cloudevents/image-processor" + r = requests.post(process_url, data=ujson.dumps(this_payload)) + + return {"result": payloads} + + +if __name__ == "__main__": + fdk.handle(handler) diff --git a/functions/scraper/func.yaml b/functions/scraper/func.yaml new file mode 100644 index 0000000..6668e64 --- /dev/null +++ b/functions/scraper/func.yaml @@ -0,0 +1,11 @@ +schema_version: 20180708 +name: scraper +version: 0.0.29 +runtime: python3.7 +entrypoint: python3 func.py +format: http-stream +timeout: 120 +triggers: +- name: scraper + type: http + source: /scraper diff --git a/functions/scraper/payload.json b/functions/scraper/payload.json new file mode 100644 index 0000000..01b9b5f --- /dev/null +++ b/functions/scraper/payload.json @@ -0,0 +1,5 @@ +{ + "query": "person", + "num": "5", + "service_to_call": "image-processor" +} diff --git a/functions/scraper/payload_faces.json b/functions/scraper/payload_faces.json new file mode 100644 index 0000000..26b9c45 --- /dev/null +++ b/functions/scraper/payload_faces.json @@ -0,0 +1,7 @@ +{ + "query": "infant smile", + "num": "50", + "countrycode": "us", + "service_to_call": "detect-faces", + "page": "5" +} diff --git a/functions/scraper/requirements.txt b/functions/scraper/requirements.txt new file mode 100644 index 0000000..af611d2 --- /dev/null +++ b/functions/scraper/requirements.txt @@ -0,0 +1,3 @@ +fdk +flickrapi +requests \ No newline at end of file diff --git a/functions/word-generator/func.yaml b/functions/word-generator/func.yaml index 1fc469f..e570a77 100644 --- a/functions/word-generator/func.yaml +++ b/functions/word-generator/func.yaml @@ -1,6 +1,6 @@ schema_version: 20180708 name: word-generator -version: 0.0.20 +version: 0.0.21 runtime: go entrypoint: ./func memory: 256 diff --git a/functions/word-generator/vendor/github.com/fnproject/fdk-go/examples/hello/func.yaml b/functions/word-generator/vendor/github.com/fnproject/fdk-go/examples/hello/func.yaml index a708e12..97b8251 100644 --- a/functions/word-generator/vendor/github.com/fnproject/fdk-go/examples/hello/func.yaml +++ b/functions/word-generator/vendor/github.com/fnproject/fdk-go/examples/hello/func.yaml @@ -1,6 +1,6 @@ schema_version: 20180708 name: hello -version: 0.0.1 +version: 0.0.2 runtime: go entrypoint: ./func format: http-stream diff --git a/setenv.sh b/setenv.sh index c1c25ca..379ddd7 100755 --- a/setenv.sh +++ b/setenv.sh @@ -8,3 +8,6 @@ fn update fn $1 image-processor --config TWITTER_ACCESS_TOKEN_KEY=$TWITTER_TOKEN fn update fn $1 image-processor --config TWITTER_ACCESS_TOKEN_SECRET=$TWITTER_TOKEN_SECRET fn update fn $1 image-processor --config SLACK_API_TOKEN=$SLACK_API_TOKEN fn update fn $1 image-processor --config SLACK_CHANNEL=$SLACK_CHANNEL +fn update fn $1 scraper --config FLICKR_API_KEY=$FLICKR_API_KEY +fn update fn $1 scraper --config FLICKR_API_SECRET=$FLICKR_API_SECRET +fn update app $1 --syslog-url $SYSLOG_URL From 70189604b9ba4b2f23bed54a97739545fd5bded2 Mon Sep 17 00:00:00 2001 From: Chad Arimura Date: Sun, 9 Dec 2018 10:49:46 -0800 Subject: [PATCH 2/2] cloudevents now --- functions/image-processor/func.py | 33 ++++---- functions/image-processor/func.yaml | 2 +- functions/image-processor/payload.ce.json | 11 +++ ...ad.sample.json => payload.sample.json.old} | 10 +-- functions/scraper/func.py | 80 ++++++++----------- functions/scraper/func.yaml | 2 +- functions/scraper/requirements.txt | 3 +- 7 files changed, 71 insertions(+), 70 deletions(-) create mode 100644 functions/image-processor/payload.ce.json rename functions/image-processor/{payload.sample.json => payload.sample.json.old} (83%) diff --git a/functions/image-processor/func.py b/functions/image-processor/func.py index 1f0db30..edbc5b3 100644 --- a/functions/image-processor/func.py +++ b/functions/image-processor/func.py @@ -326,7 +326,6 @@ def process_single_media_file(ctx, sess, media_url, label_map, event_id, event_t def with_graph(label_map): - sess = tf.Session() sess.graph.as_default() @@ -337,23 +336,23 @@ def fn(ctx, data=None, loop=None): if data is not None or len(data) !=0: data = ujson.loads(data) log.info("incoming data: {0}".format(ujson.dumps(data))) - media = data.get("media", []) - event_id = data.get("event_id") - event_type = data.get("event_type", "") - if event_type.startswith("Microsoft"): - event_type = "Azure" - event_id = event_id.replace("-", "") - ran_on = data.get("ran_on", "Fn Project on Oracle Cloud") - - for media_url in media: - img, status = process_single_media_file( - ctx, sess, media_url, label_map, - event_id, event_type, ran_on - ) - post_image(ctx, status, media_url, add_fn_logo(img)) - else: - log.info("missing data") + # media = data.get("media", []) + event_id = data.get("id") + event_type = data.get("type", "") + photo_url = data.get("data").get("photo_url") + + # if event_type.startswith("Microsoft"): + # event_type = "Azure" + # event_id = event_id.replace("-", "") + + ran_on = data.get("ran_on", "Fn Project") + + img, status = process_single_media_file( + ctx, sess, photo_url, label_map, + event_id, event_type, ran_on + ) + post_image(ctx, status, photo_url, add_fn_logo(img)) return fn diff --git a/functions/image-processor/func.yaml b/functions/image-processor/func.yaml index 257ce45..0c86a94 100644 --- a/functions/image-processor/func.yaml +++ b/functions/image-processor/func.yaml @@ -1,6 +1,6 @@ schema_version: 20180708 name: image-processor -version: 0.0.104 +version: 0.0.109 runtime: docker format: http-stream memory: 1024 diff --git a/functions/image-processor/payload.ce.json b/functions/image-processor/payload.ce.json new file mode 100644 index 0000000..1f80a4f --- /dev/null +++ b/functions/image-processor/payload.ce.json @@ -0,0 +1,11 @@ +{ + "specversion":"0.2", + "type":"cloudevent.flickr.image", + "source":"scraper", + "id":"my-id", + "contenttype":"application\/json", + "data":{ + "photo_url":"https:\/\/farm2.staticflickr.com\/1051\/1154370504_58bafbc654_c.jpg" + }, + "extensions":{} +} \ No newline at end of file diff --git a/functions/image-processor/payload.sample.json b/functions/image-processor/payload.sample.json.old similarity index 83% rename from functions/image-processor/payload.sample.json rename to functions/image-processor/payload.sample.json.old index 47b70a3..987a28e 100644 --- a/functions/image-processor/payload.sample.json +++ b/functions/image-processor/payload.sample.json.old @@ -4,15 +4,15 @@ "http://survivingchurch.org/wp-content/uploads/2016/10/Donald-Trump-Photos-HD-1024x768.png", "https://www.dairyqueen.com/Global/Food/Hot-Dogs_8-to-1_470x500.jpg", "https://pbs.twimg.com/profile_images/625802912887783424/_uzj-IZ4.jpg", - "https://i.ytimg.com/vi/lFarE1hH0ss/maxresdefault.jpg", + "https://i.ytimg.com/vi/lFarE1hH0ss/maxresdefault.jpg" + ], + "media_test": [ + "https://i.ytimg.com/vi/Lu8lDgKInSM/hqdefault.jpg", "http://m.step.aero/Burningman_2017_gifting_plane_rides.jpg", "http://m.step.aero/Burningman_2017_raising_and_rigging.jpg", "http://m.step.aero/Chad's%20Dog.png" ], - "media_test": [ - "https://i.ytimg.com/vi/Lu8lDgKInSM/hqdefault.jpg" - ], "event_id": "test_id", "event_type": "test_type", "ran_on": "" -} +} \ No newline at end of file diff --git a/functions/scraper/func.py b/functions/scraper/func.py index f6a5737..9c523b1 100644 --- a/functions/scraper/func.py +++ b/functions/scraper/func.py @@ -1,15 +1,16 @@ -import fdk import ujson import os -import random import flickrapi -import ssl -import sys +import random import requests -from fdk import fixtures +import fdk -ssl._create_default_https_context = ssl._create_unverified_context +from cloudevents.sdk import converters +from cloudevents.sdk import marshaller +from cloudevents.sdk.converters import structured +from cloudevents.sdk.event import v02 +#ssl._create_default_https_context = ssl._create_unverified_context flickr = flickrapi.FlickrAPI( os.environ.get("FLICKR_API_KEY"), @@ -18,52 +19,41 @@ format='parsed-json' ) -PHOTO_SOURCE_URL = 'https://farm{0}.staticflickr.com/{1}/{2}_{3}{4}.{5}' - -def get_image_url(photo_dict): - return PHOTO_SOURCE_URL.format( - photo_dict['farm'], photo_dict['server'], - photo_dict['id'], photo_dict['secret'], - '_c', 'jpg' - ) - - -def photo_to_payload(body, photo_dict): - return { - "id": photo_dict.get('id'), - "image_url": get_image_url(photo_dict), - "countrycode": body.get("countrycode"), - "bucket": body.get("bucket", "") - } - - def handler(ctx, data=None, loop=None): - payloads = [] + + # Scrape Flickr if data and len(data) > 0: body = ujson.loads(data) + photos = flickr.photos.search( - text=body.get("query", "baby smile"), - per_page=int(body.get("num", "5")), - page=int(body.get("page", int(random.uniform(1, 50)))), - extras="original_format", - safe_search="1", - content_type="1", + text=body.get("query", "baby smile"), + per_page=int(body.get("num", "5")), + page=int(body.get("page", int(random.uniform(1, 50)))), + extras="original_format", + safe_search="1", + content_type="1", ) + # For each photo for p in photos.get('photos', {'photo': {}}).get('photo', []): - payloads.append(photo_to_payload(body, p)) - - for p in payloads: - this_payload = { - "media":[p.get("image_url")], - "event_id": "test_id", - "event_type": "test_type", - "ran_on": "" - } - process_url = "http://docker.for.mac.localhost:8080/t/cloudevents/image-processor" - r = requests.post(process_url, data=ujson.dumps(this_payload)) - - return {"result": payloads} + photo_url_tpl = 'https://farm{0}.staticflickr.com/{1}/{2}_{3}{4}.{5}' + photo_url = photo_url_tpl.format(p['farm'], p['server'],p['id'], p['secret'], '_c', 'jpg') + + data = {"photo_url": photo_url} + + event = ( + v02.Event(). + SetContentType("application/json"). + SetData(data). + SetEventID("my-id"). + SetSource("scraper"). + SetEventType("cloudevent.flickr.image") + ) + event_json = ujson.dumps(event.Properties()) + print(event_json) + + process_url = "http://docker.for.mac.localhost:8080/t/cloudevents/image-processor" + r = requests.post(process_url, data=event_json) if __name__ == "__main__": diff --git a/functions/scraper/func.yaml b/functions/scraper/func.yaml index 6668e64..f74593d 100644 --- a/functions/scraper/func.yaml +++ b/functions/scraper/func.yaml @@ -1,6 +1,6 @@ schema_version: 20180708 name: scraper -version: 0.0.29 +version: 0.0.30 runtime: python3.7 entrypoint: python3 func.py format: http-stream diff --git a/functions/scraper/requirements.txt b/functions/scraper/requirements.txt index af611d2..444065e 100644 --- a/functions/scraper/requirements.txt +++ b/functions/scraper/requirements.txt @@ -1,3 +1,4 @@ fdk flickrapi -requests \ No newline at end of file +requests +cloudevents \ No newline at end of file