From 2deb6d3d0e88fa6187dbc2a574c4882dbd3e957f Mon Sep 17 00:00:00 2001 From: "Jens W. Klein" Date: Thu, 16 Nov 2023 20:54:07 +0100 Subject: [PATCH] local examples --- .gitignore | 2 +- examples/local/.env | 20 +++ examples/local/mappings.json | 333 +++++++++++++++++++++++++++++++++++ 3 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 examples/local/.env create mode 100644 examples/local/mappings.json diff --git a/.gitignore b/.gitignore index df70903..660dc3d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ venv/ htmlcov/ include/ lib/ -local/ +./local/ dist/* var/ # files diff --git a/examples/local/.env b/examples/local/.env new file mode 100644 index 0000000..cc7a8c2 --- /dev/null +++ b/examples/local/.env @@ -0,0 +1,20 @@ +# This env file work out-of-the-box with the opensearch example docker-compose setup as documented here: +# https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/#sample-docker-composeyml +# Be aware to increase max virtual memory map in your system before: `sysctl -w vm.max_map_count=262144` +# Follow the instructios and start the opensearch docker-compose setup. +# Then `source .env` and start the ingest-service with: celery -A collective.elastic.ingest.celery.app worker -l debug + +export MAPPINGS_FILE=example/local/mappings.json + +export INDEX_SERVER=localhost:9200 +export INDEX_USE_SSL=1 +export INDEX_OPENSEARCH=1 +export INDEX_LOGIN=admin +export INDEX_PASSWORD=admin + +export CELERY_BROKER=redis://localhost:6379/0 + +export PLONE_SERVICE=http://localhost:8080 +export PLONE_PATH=Plone +export PLONE_USER=admin +export PLONE_PASSWORD=admin diff --git a/examples/local/mappings.json b/examples/local/mappings.json new file mode 100644 index 0000000..f1fbd1b --- /dev/null +++ b/examples/local/mappings.json @@ -0,0 +1,333 @@ +{ + "behaviors/plone.allowdiscussion/allow_discussion": { + "type": "boolean" + }, + "plone.app.textfield.RichText": { + "pipeline": { + "source": "{name}__data", + "target": "{name}__extracted", + "processors": [ + { + "attachment": { + "field": "{source}", + "target_field": "{target}", + "ignore_missing": true + } + }, + { + "remove": { + "field": "{source}", + "ignore_missing": true + } + } + ], + "type": { + "type": "nested", + "dynamic": false, + "properties": { + "author": { + "type": "text" + }, + "content": { + "type": "text" + }, + "content_length": { + "type": "long" + }, + "content_type": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "title": { + "type": "text" + } + } + }, + "expansion": { + "method": "field", + "field": "data" + } + }, + "definition": { + "type": "nested", + "dynamic": false, + "properties": { + "data": { + "type": "text" + }, + "content-type": { + "type": "keyword" + }, + "encoding": { + "type": "keyword" + } + } + } + }, + "plone.namedfile.field.NamedBlobFile": { + "pipeline": { + "source": "{name}__data", + "target": "{name}__extracted", + "processors": [ + { + "attachment": { + "field": "{source}", + "target_field": "{target}", + "ignore_missing": true + } + }, + { + "remove": { + "field": "{source}", + "ignore_missing": true + } + } + ], + "type": { + "type": "nested", + "dynamic": false, + "include_in_parent": true, + "properties": { + "author": { + "type": "text" + }, + "content": { + "type": "text" + }, + "content_length": { + "type": "long" + }, + "content_type": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "title": { + "type": "text" + } + } + }, + "expansion": { + "method": "fetch", + "field": "download" + } + }, + "definition": { + "type": "nested", + "dynamic": false, + "properties": { + "content-type": { + "type": "keyword" + }, + "download": { + "type": "text" + }, + "filename": { + "type": "text" + }, + "size": { + "type": "long" + } + } + } + }, + "plone.namedfile.field.NamedBlobImage": { + "pipeline": { + "source": "{name}__data", + "target": "{name}__extracted", + "processors": [ + { + "attachment": { + "field": "{source}", + "target_field": "{target}", + "ignore_missing": true + } + }, + { + "remove": { + "field": "{source}", + "ignore_missing": true + } + } + ], + "type": { + "type": "nested", + "dynamic": false, + "properties": { + "author": { + "type": "text" + }, + "content": { + "type": "text" + }, + "content_length": { + "type": "long" + }, + "content_type": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "title": { + "type": "text" + } + } + }, + "expansion": { + "method": "fetch", + "field": "download" + } + }, + "definition": { + "type": "nested", + "dynamic": false, + "properties": { + "content-type": { + "type": "keyword" + }, + "download": { + "type": "text" + }, + "filename": { + "type": "text" + }, + "size": { + "type": "long" + }, + "height": { + "type": "long" + }, + "width": { + "type": "long" + }, + "scales": { + "type": "nested", + "dynamic": false, + "properties": { + "download": { + "type": "text" + }, + "height": { + "type": "long" + }, + "width": { + "type": "long" + } + } + } + } + } + }, + "plone.schema.jsonfield.JSONField": { + "type": "text" + }, + "z3c.relationfield.schema.RelationList": { + "type": "nested", + "dynamic": false, + "properties": { + "@id": { + "type": "keyword" + }, + "@type": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "review_state": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "z3c.relationfield.schema.RelationChoice": { + "type": "nested", + "dynamic": false, + "properties": { + "@id": { + "type": "keyword" + }, + "@type": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "review_state": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "zope.schema._bootstrapfields.Bool": { + "type": "boolean" + }, + "zope.schema._bootstrapfields.Int": { + "type": "long" + }, + "zope.schema._bootstrapfields.Text": { + "type": "text" + }, + "zope.schema._bootstrapfields.TextLine": { + "type": "text" + }, + "zope.schema._field.ASCIILine": { + "type": "keyword" + }, + "zope.schema._field.Choice": { + "type": "keyword" + }, + "zope.schema._field.Datetime": { + "type": "date" + }, + "zope.schema._field.Dict": { + "type": "object" + }, + "zope.schema._field.Float": { + "type": "double" + }, + "zope.schema._field.List": { + "type": "keyword" + }, + "zope.schema._field.Tuple": { + "type": "keyword" + }, + "zope.schema._field.URI": { + "type": "text" + } +} \ No newline at end of file