diff --git a/app/api/documents.py b/app/api/documents.py
index 706ca6e04..b4173782f 100644
--- a/app/api/documents.py
+++ b/app/api/documents.py
@@ -90,6 +90,7 @@ async def read_document(documentId: str, token=Depends(JWTBearer())):
async def delete_document(documentId: str, token=Depends(JWTBearer())):
"""Delete a document"""
try:
+ prisma.agentdocument.delete_many(where={"documentId": documentId})
prisma.document.delete(where={"id": documentId})
VectorStoreBase().get_database().delete(namespace=documentId)
return {"success": True, "data": None}
diff --git a/app/lib/documents.py b/app/lib/documents.py
index 4b4b86012..37b416631 100644
--- a/app/lib/documents.py
+++ b/app/lib/documents.py
@@ -16,9 +16,11 @@
from langchain.embeddings.openai import OpenAIEmbeddings
from llama_index.readers.schema.base import Document
+
from app.lib.parsers import CustomPDFPlumberLoader
from app.lib.splitters import TextSplitters
from app.lib.vectorstores.base import VectorStoreBase
+from app.lib.loaders.sitemap import SitemapLoader
valid_ingestion_types = [
"TXT",
@@ -29,9 +31,18 @@
"FIRESTORE",
"PSYCHIC",
"GITHUB_REPOSITORY",
+ "WEBPAGE",
+ "STRIPE",
+ "AIRTABLE",
+ "SITEMAP",
]
+def chunkify(lst, size):
+ """Divide a list into chunks of given size."""
+ return [lst[i : i + size] for i in range(0, len(lst), size)]
+
+
def upsert_document(
type: str,
document_id: str,
@@ -52,6 +63,66 @@ def upsert_document(
embeddings = OpenAIEmbeddings()
+ if type == "STRIPE":
+ pass
+
+ if type == "AIRTABLE":
+ from langchain.document_loaders import AirtableLoader
+
+ api_key = metadata["api_key"]
+ base_id = metadata["base_id"]
+ table_id = metadata["table_id"]
+ loader = AirtableLoader(api_key, table_id, base_id)
+ documents = loader.load()
+ newDocuments = [
+ document.metadata.update({"namespace": document_id}) or document
+ for document in documents
+ ]
+ docs = TextSplitters(newDocuments, text_splitter).document_splitter()
+
+ VectorStoreBase().get_database().from_documents(
+ docs, embeddings, index_name=INDEX_NAME, namespace=document_id
+ )
+
+ if type == "SITEMAP":
+ filter_urls = metadata["filter_urls"].split(",")
+ loader = SitemapLoader(sitemap_url=url, filter_urls=filter_urls)
+ documents = loader.load()
+ newDocuments = [
+ document.metadata.update({"namespace": document_id}) or document
+ for document in documents
+ ]
+ docs = TextSplitters(newDocuments, text_splitter).document_splitter()
+
+ chunk_size = 100
+ chunks = chunkify(docs, chunk_size)
+
+ for chunk in chunks:
+ VectorStoreBase().get_database().from_documents(
+ chunk, embeddings, index_name=INDEX_NAME, namespace=document_id
+ )
+
+ if type == "WEBPAGE":
+ from llama_index import download_loader
+
+ RemoteDepthReader = download_loader("RemoteDepthReader")
+ depth = int(metadata["depth"])
+ loader = RemoteDepthReader(depth=depth)
+ documents = loader.load_data(url=url)
+ langchain_documents = [d.to_langchain_format() for d in documents]
+ newDocuments = [
+ document.metadata.update({"namespace": document_id}) or document
+ for document in langchain_documents
+ ]
+ docs = TextSplitters(newDocuments, text_splitter).document_splitter()
+ chunk_size = 100
+ chunks = chunkify(docs, chunk_size)
+
+ for chunk in chunks:
+ VectorStoreBase().get_database().from_documents(
+ chunk, embeddings, index_name=INDEX_NAME, namespace=document_id
+ )
+
if type == "TXT":
file_response = content
if content is None:
diff --git a/app/lib/loaders/__init__.py b/app/lib/loaders/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/app/lib/loaders/sitemap.py b/app/lib/loaders/sitemap.py
new file mode 100644
index 000000000..979fdca5a
--- /dev/null
+++ b/app/lib/loaders/sitemap.py
@@ -0,0 +1,65 @@
+import re
+import requests
+from xml.etree import ElementTree
+from bs4 import BeautifulSoup
+from langchain.schema import Document
+
+
+class SitemapLoader:
+ SITEMAP_NAMESPACE = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
+
+ def __init__(self, sitemap_url, filter_urls=None):
+ self.sitemap_url = sitemap_url
+ self.filter_urls = filter_urls if filter_urls else []
+
+ def fetch(self, url):
+ """Fetch content of a URL using requests."""
+ response = requests.get(url)
+ response.raise_for_status() # Raise exception for HTTP errors
+ return response.text
+
+ def fetch_text(self, url):
+ response = requests.get(url)
+ response.raise_for_status() # Raise exception for HTTP errors
+ soup = BeautifulSoup(response.content, "html.parser")
+ raw_text = soup.get_text(separator=" ").strip()
+ cleaned_text = re.sub(r"\s+", " ", raw_text)
+
+ return cleaned_text
+
+ def matches_any_pattern(self, url):
+ """Check if the URL matches any of the given patterns."""
+ is_match = any(re.search(pattern, url) for pattern in self.filter_urls)
+
+ if is_match:
+ print(f"Matched URL: {url}")
+
+ return is_match
+
+ def fetch_sitemap_urls(self):
+ """Fetch URLs from a sitemap.xml file and filter based on patterns."""
+ sitemap_content = self.fetch(self.sitemap_url)
+
+ # Parse XML content
+ root = ElementTree.fromstring(sitemap_content)
+ urls = [
+ url_element.text
+ for url_element in root.findall(
+ f"{self.SITEMAP_NAMESPACE}url/{self.SITEMAP_NAMESPACE}loc"
+ )
+ ]
+
+ # Filter URLs
+ if self.filter_urls:
+ urls = [url for url in urls if self.matches_any_pattern(url)]
+
+ return urls
+
+ def load(self):
+ """Fetch content of each URL listed in a sitemap.xml file."""
+ urls = self.fetch_sitemap_urls()
+
+ return [
+ Document(page_content=self.fetch_text(url), metadata={"url": url})
+ for url in urls
+ ]
diff --git a/poetry.lock b/poetry.lock
index 76f16e319..0ea5d644f 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -122,6 +122,74 @@ files = [
[package.dependencies]
frozenlist = ">=1.1.0"
+[[package]]
+name = "airbyte-cdk"
+version = "0.50.2"
+description = "A framework for writing Airbyte Connectors."
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "airbyte-cdk-0.50.2.tar.gz", hash = "sha256:de765e720e446758b436aff360c36c33f8b085be46267422d1a4b743d3da1566"},
+ {file = "airbyte_cdk-0.50.2-py3-none-any.whl", hash = "sha256:d0a304fd6659ea36391e88f6a6c7056af1ca149922ae97c2662656f05767d793"},
+]
+
+[package.dependencies]
+airbyte-protocol-models = "0.4.0"
+backoff = "*"
+cachetools = "*"
+Deprecated = ">=1.2,<2.0"
+dpath = ">=2.0.1,<2.1.0"
+genson = "1.2.2"
+isodate = ">=0.6.1,<0.7.0"
+Jinja2 = ">=3.1.2,<3.2.0"
+jsonref = ">=0.2,<1.0"
+jsonschema = ">=3.2.0,<3.3.0"
+pendulum = "*"
+pydantic = ">=1.9.2,<2.0.0"
+python-dateutil = "*"
+PyYAML = ">=6.0.1"
+requests = "*"
+requests-cache = "*"
+wcmatch = "8.4"
+
+[package.extras]
+dev = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "mypy", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock"]
+file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"]
+sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"]
+
+[[package]]
+name = "airbyte-protocol-models"
+version = "0.4.0"
+description = "Declares the Airbyte Protocol."
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "airbyte_protocol_models-0.4.0-py3-none-any.whl", hash = "sha256:e6a31fcd237504198a678d02c0040a8798f281c39203da61a5abce67842c5360"},
+ {file = "airbyte_protocol_models-0.4.0.tar.gz", hash = "sha256:518736015c29ac60b6b8964a1b0d9b52e40020bcbd89e2545cc781f0b37d0f2b"},
+]
+
+[package.dependencies]
+pydantic = ">=1.9.2,<2.0.0"
+
+[[package]]
+name = "airbyte-source-stripe"
+version = "3.17.1"
+description = "Source implementation for Stripe."
+optional = false
+python-versions = "*"
+files = [
+ {file = "airbyte-source-stripe-3.17.1.tar.gz", hash = "sha256:7c653ae0ed788766e706db7a8d35a25e3602c9af15d52895724738154b48780e"},
+ {file = "airbyte_source_stripe-3.17.1-py3-none-any.whl", hash = "sha256:8cf99fd3095bc726fdbf6ca82d39c3ce4d426349329c8c1ce684ef7ef2f059af"},
+]
+
+[package.dependencies]
+airbyte-cdk = "*"
+pendulum = "2.1.2"
+stripe = "2.56.0"
+
+[package.extras]
+tests = ["connector-acceptance-test", "pytest (>=6.1,<7.0)", "requests-mock", "requests-mock (>=1.8,<2.0)"]
+
[[package]]
name = "anyio"
version = "3.6.2"
@@ -383,6 +451,17 @@ d = ["aiohttp (>=3.7.4)"]
jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
uvloop = ["uvloop (>=0.15.2)"]
+[[package]]
+name = "bracex"
+version = "2.3.post1"
+description = "Bash style brace expander."
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "bracex-2.3.post1-py3-none-any.whl", hash = "sha256:351b7f20d56fb9ea91f9b9e9e7664db466eb234188c175fd943f8f755c807e73"},
+ {file = "bracex-2.3.post1.tar.gz", hash = "sha256:e7b23fc8b2cd06d3dec0692baabecb249dda94e06a617901ff03a6c56fd71693"},
+]
+
[[package]]
name = "bs4"
version = "0.0.1"
@@ -407,6 +486,31 @@ files = [
{file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"},
]
+[[package]]
+name = "cattrs"
+version = "23.1.2"
+description = "Composable complex class support for attrs and dataclasses."
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"},
+ {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"},
+]
+
+[package.dependencies]
+attrs = ">=20"
+exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""}
+
+[package.extras]
+bson = ["pymongo (>=4.2.0,<5.0.0)"]
+cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"]
+msgpack = ["msgpack (>=1.0.2,<2.0.0)"]
+orjson = ["orjson (>=3.5.2,<4.0.0)"]
+pyyaml = ["PyYAML (>=6.0,<7.0)"]
+tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"]
+ujson = ["ujson (>=5.4.0,<6.0.0)"]
+
[[package]]
name = "certifi"
version = "2023.5.7"
@@ -747,6 +851,17 @@ idna = ["idna (>=2.1,<4.0)"]
trio = ["trio (>=0.14,<0.23)"]
wmi = ["wmi (>=1.5.1,<2.0.0)"]
+[[package]]
+name = "dpath"
+version = "2.0.8"
+description = "Filesystem-like pathing and searching for dictionaries"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"},
+ {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"},
+]
+
[[package]]
name = "email-validator"
version = "2.0.0.post2"
@@ -773,6 +888,20 @@ files = [
{file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"},
]
+[[package]]
+name = "exceptiongroup"
+version = "1.1.2"
+description = "Backport of PEP 654 (exception groups)"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"},
+ {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"},
+]
+
+[package.extras]
+test = ["pytest (>=6)"]
+
[[package]]
name = "fastapi"
version = "0.95.1"
@@ -952,6 +1081,16 @@ smb = ["smbprotocol"]
ssh = ["paramiko"]
tqdm = ["tqdm"]
+[[package]]
+name = "genson"
+version = "1.2.2"
+description = "GenSON is a powerful, user-friendly JSON Schema generator."
+optional = false
+python-versions = "*"
+files = [
+ {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"},
+]
+
[[package]]
name = "gitdb"
version = "4.0.10"
@@ -1436,6 +1575,31 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker
perf = ["ipython"]
testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"]
+[[package]]
+name = "inflection"
+version = "0.5.1"
+description = "A port of Ruby on Rails inflector to Python"
+optional = false
+python-versions = ">=3.5"
+files = [
+ {file = "inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2"},
+ {file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"},
+]
+
+[[package]]
+name = "isodate"
+version = "0.6.1"
+description = "An ISO 8601 date/time/duration parser and formatter"
+optional = false
+python-versions = "*"
+files = [
+ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},
+ {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},
+]
+
+[package.dependencies]
+six = "*"
+
[[package]]
name = "itsdangerous"
version = "2.1.2"
@@ -1490,15 +1654,47 @@ files = [
{file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"},
]
+[[package]]
+name = "jsonref"
+version = "0.3.0"
+description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python."
+optional = false
+python-versions = ">=3.3,<4.0"
+files = [
+ {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"},
+ {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"},
+]
+
+[[package]]
+name = "jsonschema"
+version = "3.2.0"
+description = "An implementation of JSON Schema validation for Python"
+optional = false
+python-versions = "*"
+files = [
+ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"},
+ {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"},
+]
+
+[package.dependencies]
+attrs = ">=17.4.0"
+pyrsistent = ">=0.14.0"
+setuptools = "*"
+six = ">=1.11.0"
+
+[package.extras]
+format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"]
+format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"]
+
[[package]]
name = "langchain"
-version = "0.0.252"
+version = "0.0.260"
description = "Building applications with LLMs through composability"
optional = false
python-versions = ">=3.8.1,<4.0"
files = [
- {file = "langchain-0.0.252-py3-none-any.whl", hash = "sha256:fb14d2346f453656e8f6c138da9123918796ae950a26606c91c156f2e5c0f8c4"},
- {file = "langchain-0.0.252.tar.gz", hash = "sha256:1d205417793bbf9410e0eb23d14b5ce6f13946ccf26a05dcc19bb10def92b03d"},
+ {file = "langchain-0.0.260-py3-none-any.whl", hash = "sha256:5ad68342bde9bc5d4590bb042c3fbcfdf800053750e38b621e7f26c88a15e3dc"},
+ {file = "langchain-0.0.260.tar.gz", hash = "sha256:e9a5c3dca51806b7829abaf8406cb9b17a8ab8fba05ffc6350ef92a813b5db8d"},
]
[package.dependencies]
@@ -1522,11 +1718,12 @@ clarifai = ["clarifai (>=9.1.0)"]
cohere = ["cohere (>=4,<5)"]
docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"]
embeddings = ["sentence-transformers (>=2,<3)"]
-extended-testing = ["atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.0.7,<0.0.8)", "chardet (>=5.1.0,<6.0.0)", "esprima (>=4.0.1,<5.0.0)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "xinference (>=0.0.6,<0.0.7)", "zep-python (>=0.32)"]
+extended-testing = ["amazon-textract-caller (<2)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.0.7,<0.0.8)", "chardet (>=5.1.0,<6.0.0)", "esprima (>=4.0.1,<5.0.0)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "xata (>=1.0.0a7,<2.0.0)", "xinference (>=0.0.6,<0.0.7)", "xmltodict (>=0.13.0,<0.14.0)", "zep-python (>=0.32)"]
javascript = ["esprima (>=4.0.1,<5.0.0)"]
llms = ["anthropic (>=0.3,<0.4)", "clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openllm (>=0.1.19)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)", "xinference (>=0.0.6,<0.0.7)"]
openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.4.0)"]
qdrant = ["qdrant-client (>=1.3.1,<2.0.0)"]
+scheduled-testing = ["openai (>=0,<1)"]
text-helpers = ["chardet (>=5.1.0,<6.0.0)"]
[[package]]
@@ -2344,6 +2541,40 @@ files = [
Pillow = ">=9.1"
Wand = ">=0.6.10"
+[[package]]
+name = "pendulum"
+version = "2.1.2"
+description = "Python datetimes made easy"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+ {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"},
+ {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"},
+ {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"},
+ {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"},
+ {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"},
+ {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"},
+ {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"},
+ {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"},
+ {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"},
+ {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"},
+ {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"},
+ {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"},
+ {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"},
+ {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"},
+ {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"},
+ {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"},
+ {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"},
+ {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"},
+ {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"},
+ {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"},
+ {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"},
+]
+
+[package.dependencies]
+python-dateutil = ">=2.6,<3.0"
+pytzdata = ">=2020.1"
+
[[package]]
name = "pillow"
version = "9.5.0"
@@ -2586,6 +2817,24 @@ files = [
[package.dependencies]
requests = "*"
+[[package]]
+name = "pyairtable"
+version = "2.0.0"
+description = "Python Client for the Airtable API"
+optional = false
+python-versions = "*"
+files = [
+ {file = "pyairtable-2.0.0-py2.py3-none-any.whl", hash = "sha256:41e256fdcf0e68e8348197d8f474b0789b6f05d2fd3bd6c407e5900a8fae02f8"},
+ {file = "pyairtable-2.0.0.tar.gz", hash = "sha256:dd2dde960f261f947407cf865922bcf2e0e2524b94c7a596c8901dfdf4471b68"},
+]
+
+[package.dependencies]
+inflection = "*"
+pydantic = ">=1.10,<2.0"
+requests = ">=2.22.0"
+typing-extensions = "*"
+urllib3 = ">=1.26"
+
[[package]]
name = "pyasn1"
version = "0.5.0"
@@ -2805,6 +3054,42 @@ docs = ["myst_parser", "sphinx", "sphinx_rtd_theme"]
full = ["Pillow", "PyCryptodome"]
image = ["Pillow"]
+[[package]]
+name = "pyrsistent"
+version = "0.19.3"
+description = "Persistent/Functional/Immutable data structures"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"},
+ {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"},
+ {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"},
+ {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"},
+ {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"},
+ {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"},
+ {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"},
+ {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"},
+ {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"},
+ {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"},
+ {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"},
+ {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"},
+ {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"},
+ {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"},
+ {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"},
+ {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"},
+ {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"},
+ {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"},
+ {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"},
+ {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"},
+ {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"},
+ {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"},
+ {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"},
+ {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"},
+ {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"},
+ {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"},
+ {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"},
+]
+
[[package]]
name = "python-dateutil"
version = "2.8.2"
@@ -2925,6 +3210,17 @@ files = [
{file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"},
]
+[[package]]
+name = "pytzdata"
+version = "2020.1"
+description = "The Olson timezone database for Python."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+ {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"},
+ {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"},
+]
+
[[package]]
name = "pywin32"
version = "306"
@@ -2950,51 +3246,51 @@ files = [
[[package]]
name = "pyyaml"
-version = "6.0"
+version = "6.0.1"
description = "YAML parser and emitter for Python"
optional = false
python-versions = ">=3.6"
files = [
- {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
- {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
- {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"},
- {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"},
- {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"},
- {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"},
- {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"},
- {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"},
- {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"},
- {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"},
- {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"},
- {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"},
- {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"},
- {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"},
- {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"},
- {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"},
- {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"},
- {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"},
- {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"},
- {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"},
- {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"},
- {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"},
- {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"},
- {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"},
- {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"},
- {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"},
- {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"},
- {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"},
- {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"},
- {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"},
- {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"},
- {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"},
- {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"},
- {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"},
- {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"},
- {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"},
- {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"},
- {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"},
- {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
- {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
+ {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
+ {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
+ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
+ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
+ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
+ {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
+ {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
+ {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
+ {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
+ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
+ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
+ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
+ {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
+ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
+ {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
+ {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
+ {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
+ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
+ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
+ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
+ {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
+ {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
+ {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
+ {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
+ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
+ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
+ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
+ {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
+ {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
+ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
]
[[package]]
@@ -3134,6 +3430,36 @@ urllib3 = ">=1.21.1,<1.27"
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+[[package]]
+name = "requests-cache"
+version = "1.1.0"
+description = "A persistent cache for python requests"
+optional = false
+python-versions = ">=3.7,<4.0"
+files = [
+ {file = "requests_cache-1.1.0-py3-none-any.whl", hash = "sha256:178282bce704b912c59e7f88f367c42bddd6cde6bf511b2a3e3cfb7e5332a92a"},
+ {file = "requests_cache-1.1.0.tar.gz", hash = "sha256:41b79166aa8e300cc4de982f7ab7c52af914a785160be1eda25c6e9265969a67"},
+]
+
+[package.dependencies]
+attrs = ">=21.2"
+cattrs = ">=22.2"
+platformdirs = ">=2.5"
+requests = ">=2.22"
+url-normalize = ">=1.4"
+urllib3 = ">=1.25.5"
+
+[package.extras]
+all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"]
+bson = ["bson (>=0.5)"]
+docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"]
+dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"]
+json = ["ujson (>=5.4)"]
+mongodb = ["pymongo (>=3)"]
+redis = ["redis (>=3)"]
+security = ["itsdangerous (>=2.0)"]
+yaml = ["pyyaml (>=5.4)"]
+
[[package]]
name = "requests-oauthlib"
version = "1.3.1"
@@ -3471,6 +3797,20 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""
[package.extras]
full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"]
+[[package]]
+name = "stripe"
+version = "2.56.0"
+description = "Python bindings for the Stripe API"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+ {file = "stripe-2.56.0-py2.py3-none-any.whl", hash = "sha256:6c685eeadf9e3608315b6d84b4f5f2da2909179b65633ce20f296be22ed21a98"},
+ {file = "stripe-2.56.0.tar.gz", hash = "sha256:2ff904fb8dee0d25f135059468a876852d24dc8cbe0b45d7aff56a028045777c"},
+]
+
+[package.dependencies]
+requests = {version = ">=2.20", markers = "python_version >= \"3.0\""}
+
[[package]]
name = "superagent-py"
version = "0.0.29"
@@ -3907,6 +4247,20 @@ s3 = ["fsspec", "s3fs"]
slack = ["slack-sdk"]
wikipedia = ["wikipedia"]
+[[package]]
+name = "url-normalize"
+version = "1.4.3"
+description = "URL normalization for Python"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+ {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"},
+ {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"},
+]
+
+[package.dependencies]
+six = "*"
+
[[package]]
name = "urllib3"
version = "1.26.15"
@@ -4065,6 +4419,20 @@ files = [
[package.dependencies]
anyio = ">=3.0.0"
+[[package]]
+name = "wcmatch"
+version = "8.4"
+description = "Wildcard/glob file name matcher."
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"},
+ {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"},
+]
+
+[package.dependencies]
+bracex = ">=2.1.1"
+
[[package]]
name = "websockets"
version = "11.0.3"
@@ -4392,4 +4760,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
[metadata]
lock-version = "2.0"
python-versions = "^3.9"
-content-hash = "3b0241a216ca44aa311cad1e14422cfcfa31f6ea983e4cb08147cbd31daa54e5"
+content-hash = "2c070bb153ca9335ee343655ed8bd43f4d3e189a4d2600858769b07faa915372"
diff --git a/prisma/migrations/20230808152201_document_webpage/migration.sql b/prisma/migrations/20230808152201_document_webpage/migration.sql
new file mode 100644
index 000000000..86878b2c8
--- /dev/null
+++ b/prisma/migrations/20230808152201_document_webpage/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DocumentType" ADD VALUE 'WEBPAGE';
diff --git a/prisma/migrations/20230809165852_document_stripe/migration.sql b/prisma/migrations/20230809165852_document_stripe/migration.sql
new file mode 100644
index 000000000..ae76a3f85
--- /dev/null
+++ b/prisma/migrations/20230809165852_document_stripe/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DocumentType" ADD VALUE 'STRIPE';
diff --git a/prisma/migrations/20230809204553_document_airtable/migration.sql b/prisma/migrations/20230809204553_document_airtable/migration.sql
new file mode 100644
index 000000000..2e4592703
--- /dev/null
+++ b/prisma/migrations/20230809204553_document_airtable/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DocumentType" ADD VALUE 'AIRTABLE';
diff --git a/prisma/migrations/20230810071652_document_sitemap/migration.sql b/prisma/migrations/20230810071652_document_sitemap/migration.sql
new file mode 100644
index 000000000..9f30b7285
--- /dev/null
+++ b/prisma/migrations/20230810071652_document_sitemap/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DocumentType" ADD VALUE 'SITEMAP';
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index 46ca55609..077b8d7d7 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -31,6 +31,10 @@ enum DocumentType {
FIRESTORE
PSYCHIC
GITHUB_REPOSITORY
+ WEBPAGE
+ STRIPE
+ AIRTABLE
+ SITEMAP
}
enum ToolType {
diff --git a/pyproject.toml b/pyproject.toml
index 52fd9cf23..56ef2ff26 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,8 +49,10 @@ colorlog = "^6.7.0"
llama-hub = "^0.0.16"
pygithub = "^1.59.0"
gitpython = "^3.1.32"
-langchain = "^0.0.252"
vulture = "^2.7"
+airbyte-source-stripe = "^3.17.1"
+pyairtable = "^2.0.0"
+langchain = "^0.0.260"
[build-system]
diff --git a/ui/.env.example b/ui/.env.example
index 9cfbe7039..c3e38440a 100644
--- a/ui/.env.example
+++ b/ui/.env.example
@@ -1,10 +1,16 @@
NEXTAUTH_URL="http://localhost:3000"
NEXTAUTH_SECRET="superagent"
NEXT_PUBLIC_SUPERAGENT_API_URL="http://127.0.0.1:8080/api/v1"
+# Mandatory for uploading files
+NEXT_PUBLIC_AWS_S3_BUCKET=
+NEXT_PUBLIC_AWS_S3_REGION=
+NEXT_PUBLIC_AMAZON_S3_ACCESS_KEY_ID=
+NEXT_PUBLIC_AMAZON_S3_SECRET_ACCESS_KEY=
+
# Shareable Key Needs to be exactly 16 characters, do not wrap value in quotes
NEXT_PUBLIC_SHARABLE_KEY_SECRET=
-# Optiona OAuth values
+# Optional OAuth values
NEXT_PUBLIC_GITHUB_CLIENT_ID=""
NEXT_PUBLIC_GITHUB_CLIENT_SECRET=""
NEXT_PUBLIC_GOOGLE_CLIENT_ID=""
diff --git a/ui/app/_components/search-bar.js b/ui/app/_components/search-bar.js
index e7c3c33af..9375ea069 100644
--- a/ui/app/_components/search-bar.js
+++ b/ui/app/_components/search-bar.js
@@ -47,7 +47,7 @@ export default function SearchBar({ onSearch, onReset }) {
{...register("searchTerm", { required: true })}
/>
- {searchTerm && (
+ {searchTerm && onReset && (
<>
}
diff --git a/ui/app/agents/[agentId]/client-page.js b/ui/app/agents/[agentId]/client-page.js
index edba337fb..ad286cd91 100644
--- a/ui/app/agents/[agentId]/client-page.js
+++ b/ui/app/agents/[agentId]/client-page.js
@@ -36,7 +36,7 @@ import { useAsyncFn } from "react-use";
import { TOOL_ICONS } from "@/lib/constants";
import API from "@/lib/api";
import AgentNavbar from "./_components/nav";
-import DocumentPickerModal from "@/app/documents/document-picker";
+import DocumentPickerModal from "@/app/datasources/_components/document-picker";
import ToolPickerModal from "@/app/tools/tool-picker";
import TagPickerModal from "@/app/tags/tag-picker";
import { getPromptVariables } from "@/lib/prompts";
diff --git a/ui/app/datasources/_components/applications.js b/ui/app/datasources/_components/applications.js
new file mode 100644
index 000000000..5b22af5bc
--- /dev/null
+++ b/ui/app/datasources/_components/applications.js
@@ -0,0 +1,507 @@
+"use client";
+import { useCallback, useState } from "react";
+import SearchBar from "@/app/_components/search-bar";
+import { useRouter } from "next/navigation";
+import {
+ Button,
+ Box,
+ Card,
+ CardBody,
+ CardFooter,
+ HStack,
+ FormControl,
+ FormLabel,
+ FormHelperText,
+ Input,
+ Spinner,
+ Icon,
+ Modal,
+ ModalBody,
+ ModalOverlay,
+ ModalHeader,
+ ModalContent,
+ ModalCloseButton,
+ Stack,
+ Text,
+ Table,
+ Thead,
+ Tbody,
+ Tr,
+ Th,
+ Td,
+ TableContainer,
+ IconButton,
+ useToast,
+ useDisclosure,
+ Tag,
+ SimpleGrid,
+ Avatar,
+ Select,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import { useForm } from "react-hook-form";
+import { TbCopy, TbPlus, TbTrash, TbPencil } from "react-icons/tb";
+import relativeTime from "dayjs/plugin/relativeTime";
+import { useAsyncFn } from "react-use";
+import API from "@/lib/api";
+import { APPLICATIONS, ACCEPTABLE_APPLICATION_TYPES } from "@/lib/datasources";
+
+dayjs.extend(relativeTime);
+
+function DocumentRow({ id, name, createdAt, type, onDelete, onEdit }) {
+ const toast = useToast();
+ const copyToClipboard = (text) => {
+ navigator.clipboard.writeText(text);
+
+ toast({
+ description: "Copied to clipboard",
+ position: "top",
+ colorScheme: "gray",
+ });
+ };
+
+ const [{ loading: isDeleting }, handleDelete] = useAsyncFn(
+ async (id) => {
+ await onDelete(id);
+
+ toast({
+ description: "Document deleted",
+ position: "top",
+ colorScheme: "gray",
+ });
+ },
+ [onDelete]
+ );
+
+ return (
+
+ |
+
+ {name}
+
+ |
+
+
+ {type}
+
+ |
+
+
+ {id}
+ }
+ onClick={() => copyToClipboard(id)}
+ />
+
+ |
+
+ {dayjs(createdAt).fromNow()}
+ |
+
+
+ }
+ onClick={() => onEdit(id)}
+ />
+
+
+ ) : (
+
+ )
+ }
+ onClick={() => handleDelete(id)}
+ />
+
+ |
+
+ );
+}
+
+export default function Applications({ data, session }) {
+ const [filteredData, setData] = useState();
+ const [filteredAppData, setAppData] = useState();
+ const [selectedSource, setSelectedSource] = useState();
+ const { isOpen, onClose, onOpen } = useDisclosure();
+ const [selectedDocument, setSelectedDocument] = useState();
+ const router = useRouter();
+ const api = new API(session);
+ const toast = useToast();
+ const {
+ formState: { isSubmitting },
+ handleSubmit,
+ register,
+ reset,
+ setValue,
+ watch,
+ } = useForm();
+ const url = watch("url");
+
+ const onCancel = async () => {
+ reset();
+ setSelectedDocument();
+ setSelectedSource();
+ setAppData();
+ onClose();
+ };
+
+ const handleSearch = ({ searchTerm }) => {
+ if (!searchTerm) {
+ setData(data);
+ }
+
+ const keysToFilter = ["name", "type"];
+ const filteredItems = data.filter((item) =>
+ keysToFilter.some((key) =>
+ item[key].toString().toLowerCase().includes(searchTerm.toLowerCase())
+ )
+ );
+
+ setData(filteredItems);
+ };
+
+ const handleAppSearch = ({ searchTerm }) => {
+ if (searchTerm.length < 3) {
+ setAppData(APPLICATIONS);
+ return;
+ }
+
+ const keysToFilter = ["name"];
+ const filteredItems = APPLICATIONS.filter((item) =>
+ keysToFilter.some((key) =>
+ item[key].toString().toLowerCase().includes(searchTerm.toLowerCase())
+ )
+ );
+
+ setAppData(filteredItems);
+ };
+
+ const handleDelete = async (id) => {
+ await api.deleteDocument({ id });
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Deleted Document", { id });
+ }
+
+ setData();
+ router.refresh();
+ };
+
+ const onSubmit = useCallback(
+ async (values) => {
+ const { name, description, url, ...metadata } = values;
+ const payload = {
+ name: name,
+ description,
+ url,
+ type: selectedSource.id,
+ metadata,
+ };
+
+ await api.createDocument(payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Connected application", { ...payload });
+ }
+
+ toast({
+ description: "Application connected",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ onCancel();
+
+ router.refresh();
+ },
+ [selectedSource, router, onCancel]
+ );
+
+ const handleEdit = async (documentId) => {
+ const document = data.find(({ id }) => id === documentId);
+ const selectedSource = APPLICATIONS.find(({ id }) => id === document.type);
+
+ setSelectedDocument(documentId);
+ setSelectedSource(selectedSource);
+ setValue("name", document?.name);
+ setValue("description", document?.description);
+ onOpen();
+ };
+
+ const onUpdate = async (values) => {
+ const { name, description } = values;
+ const payload = {
+ name,
+ description,
+ };
+
+ await api.patchDocument(selectedDocument, payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Updated Application", { ...payload });
+ }
+
+ toast({
+ description: "Application updated",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ setData();
+ router.refresh();
+ reset();
+ setSelectedDocument();
+ onCancel();
+ };
+
+ return (
+
+
+
+ } onClick={onOpen}>
+ Add application
+
+
+ Import data from Intercom, Hubspot, Zendesk, Jira, Notion and many
+ more...
+
+
+
+
+ handleSearch(values)}
+ onReset={() => setData(data)}
+ />
+
+
+
+
+ | Name |
+ Type |
+ ID |
+ Created at |
+ |
+
+
+
+ {filteredData
+ ? filteredData
+ ?.filter(({ type }) =>
+ ACCEPTABLE_APPLICATION_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))
+ : data
+ ?.filter(({ type }) =>
+ ACCEPTABLE_APPLICATION_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))}
+
+
+
+
+
+
+
+
+ {selectedSource?.name || "Connect application"}
+
+
+
+
+ {selectedSource ? (
+
+
+
+ Name
+
+
+
+ Description
+
+
+ Use this to instruct the Agent when to query this
+ source.
+
+
+ {!selectedDocument &&
+ selectedSource.inputs.map(
+ ({ key, name, type, required, options, helpText }) => (
+
+ {name}
+ {type === "input" && (
+
+ )}
+ {type === "date" && (
+
+ )}
+ {type === "select" && }
+ {helpText && (
+ {helpText}
+ )}
+
+ )
+ )}
+
+ {selectedSource && (
+
+
+
+
+ )}
+
+ ) : (
+
+ handleAppSearch(values)} />
+
+ {filteredAppData
+ ? filteredAppData.map(({ name, logo, id, is_live }) => (
+
+
+
+
+ {name}
+
+
+
+ }
+ onClick={() =>
+ setSelectedSource(
+ APPLICATIONS.find(
+ ({ id: appId }) => appId === id
+ )
+ )
+ }
+ >
+ {is_live ? "Add" : "Coming soon"}
+
+
+
+ ))
+ : APPLICATIONS.map(({ name, logo, id, is_live }) => (
+
+
+
+
+ {name}
+
+
+
+ }
+ onClick={() =>
+ setSelectedSource(
+ APPLICATIONS.find(
+ ({ id: appId }) => appId === id
+ )
+ )
+ }
+ >
+ {is_live ? "Add" : "Coming soon"}
+
+
+
+ ))}
+
+
+ )}
+
+
+
+
+
+ );
+}
diff --git a/ui/app/documents/document-picker.js b/ui/app/datasources/_components/document-picker.js
similarity index 100%
rename from ui/app/documents/document-picker.js
rename to ui/app/datasources/_components/document-picker.js
diff --git a/ui/app/datasources/_components/files.js b/ui/app/datasources/_components/files.js
new file mode 100644
index 000000000..22b10fdab
--- /dev/null
+++ b/ui/app/datasources/_components/files.js
@@ -0,0 +1,371 @@
+"use client";
+import { useCallback, useState } from "react";
+import SearchBar from "@/app/_components/search-bar";
+import { useRouter } from "next/navigation";
+import {
+ Button,
+ Code,
+ HStack,
+ FormControl,
+ FormLabel,
+ FormHelperText,
+ Input,
+ Spinner,
+ Icon,
+ Modal,
+ ModalBody,
+ ModalOverlay,
+ ModalHeader,
+ ModalContent,
+ ModalCloseButton,
+ ModalFooter,
+ Stack,
+ Text,
+ Table,
+ Thead,
+ Tbody,
+ Tr,
+ Th,
+ Td,
+ TableContainer,
+ IconButton,
+ useToast,
+ useDisclosure,
+ Tag,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import { useForm } from "react-hook-form";
+import { TbCopy, TbTrash, TbPencil } from "react-icons/tb";
+import relativeTime from "dayjs/plugin/relativeTime";
+import { useAsyncFn } from "react-use";
+import API from "@/lib/api";
+import UploadButton from "./upload-button";
+import {
+ ACCEPTABLE_STATIC_FILE_TYPES,
+ getFileType,
+ uploadFile,
+} from "@/lib/datasources";
+
+dayjs.extend(relativeTime);
+
+function DocumentRow({ id, name, createdAt, type, onDelete, onEdit }) {
+ const toast = useToast();
+ const copyToClipboard = (text) => {
+ navigator.clipboard.writeText(text);
+
+ toast({
+ description: "Copied to clipboard",
+ position: "top",
+ colorScheme: "gray",
+ });
+ };
+
+ const [{ loading: isDeleting }, handleDelete] = useAsyncFn(
+ async (id) => {
+ await onDelete(id);
+
+ toast({
+ description: "Document deleted",
+ position: "top",
+ colorScheme: "gray",
+ });
+ },
+ [onDelete]
+ );
+
+ return (
+
+ |
+
+ {name}
+
+ |
+
+
+ {type}
+
+ |
+
+
+ {id}
+ }
+ onClick={() => copyToClipboard(id)}
+ />
+
+ |
+
+ {dayjs(createdAt).fromNow()}
+ |
+
+
+ }
+ onClick={() => onEdit(id)}
+ />
+
+
+ ) : (
+
+ )
+ }
+ onClick={() => handleDelete(id)}
+ />
+
+ |
+
+ );
+}
+
+export default function Files({ data, session }) {
+ const [filteredData, setData] = useState();
+ const [isCreatingDocument, setIsCreatingDocument] = useState();
+ const { isOpen, onClose, onOpen } = useDisclosure();
+ const [selectedDocument, setSelectedDocument] = useState();
+ const router = useRouter();
+ const api = new API(session);
+ const toast = useToast();
+ const {
+ formState: { isSubmitting, errors },
+ handleSubmit,
+ register,
+ reset,
+ setValue,
+ } = useForm();
+ const onCancel = async () => {
+ reset();
+ setSelectedDocument();
+ onClose();
+ };
+
+ const onUpdate = async (values) => {
+ const { name, description } = values;
+ const payload = {
+ name,
+ description,
+ };
+
+ await api.patchDocument(selectedDocument, payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Updated Document", { ...payload });
+ }
+
+ toast({
+ description: "Document updated",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ setData();
+ router.refresh();
+ reset();
+ setSelectedDocument();
+ onClose();
+ };
+
+ const handleSearch = ({ searchTerm }) => {
+ if (!searchTerm) {
+ setData(data);
+ }
+
+ const keysToFilter = ["name", "type"];
+ const filteredItems = data.filter((item) =>
+ keysToFilter.some((key) =>
+ item[key].toString().toLowerCase().includes(searchTerm.toLowerCase())
+ )
+ );
+
+ setData(filteredItems);
+ };
+
+ const handleDelete = async (id) => {
+ await api.deleteDocument({ id });
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Deleted Document", { id });
+ }
+
+ setData();
+ router.refresh();
+ };
+
+ const handleEdit = async (documentId) => {
+ const document = data.find(({ id }) => id === documentId);
+
+ setSelectedDocument(documentId);
+ setValue("name", document?.name);
+ setValue("description", document?.description);
+ onOpen();
+ };
+
+ const onUpload = useCallback(async (file) => {
+ const type = getFileType(file.type);
+ const { Location } = await uploadFile(file);
+ const payload = {
+ name: file.name,
+ description: `Useful for answering questions about ${file.name}`,
+ type,
+ url: Location,
+ };
+
+ await api.createDocument(payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Created Document", { ...payload });
+ }
+
+ toast({
+ description: "Document created",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ router.refresh();
+ }, []);
+
+ return (
+
+
+
+
+
+ We currently support .txt, .pdf,{" "}
+ .csv and .md files
+
+
+
+
+ handleSearch(values)}
+ onReset={() => setData(data)}
+ />
+
+
+
+
+ | Name |
+ Type |
+ ID |
+ Created at |
+ |
+
+
+
+ {filteredData
+ ? filteredData
+ ?.filter(({ type }) =>
+ ACCEPTABLE_STATIC_FILE_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))
+ : data
+ ?.filter(({ type }) =>
+ ACCEPTABLE_STATIC_FILE_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))}
+
+
+
+
+
+
+
+ Update document
+
+
+
+
+
+ Name
+
+ A document name.
+ {errors?.name && (
+ Invalid name
+ )}
+
+
+ Description
+
+
+ What is this document useful for?
+
+ {errors?.description && (
+ Invalid description
+ )}
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/ui/app/datasources/_components/upload-button.js b/ui/app/datasources/_components/upload-button.js
new file mode 100644
index 000000000..c0a7178cd
--- /dev/null
+++ b/ui/app/datasources/_components/upload-button.js
@@ -0,0 +1,44 @@
+"use client";
+import { useState, useRef } from "react";
+import { Button, Icon, Input } from "@chakra-ui/react";
+import { TbPlus } from "react-icons/tb";
+import { useAsyncFn } from "react-use";
+
+export default function UploadButton({ accept, label, onSelect }) {
+ const fileInputRef = useRef(null);
+ const [isLoading, setIsLoading] = useState();
+
+ const handleFileChange = async (event) => {
+ const file = event.target.files[0];
+ setIsLoading(true);
+
+ if (file) {
+ await onSelect(file);
+ setIsLoading();
+ }
+ };
+
+ const triggerFileInput = () => {
+ fileInputRef.current.click();
+ };
+
+ return (
+ <>
+
+ }
+ onClick={triggerFileInput}
+ isLoading={isLoading}
+ loadingText="Working..."
+ >
+ {label}
+
+
+ );
+}
diff --git a/ui/app/datasources/_components/webpages.js b/ui/app/datasources/_components/webpages.js
new file mode 100644
index 000000000..d412d610d
--- /dev/null
+++ b/ui/app/datasources/_components/webpages.js
@@ -0,0 +1,421 @@
+"use client";
+import { useCallback, useState } from "react";
+import SearchBar from "@/app/_components/search-bar";
+import { useRouter } from "next/navigation";
+import {
+ Button,
+ Code,
+ HStack,
+ FormControl,
+ FormLabel,
+ FormHelperText,
+ Input,
+ Spinner,
+ Icon,
+ Modal,
+ ModalBody,
+ ModalOverlay,
+ ModalHeader,
+ ModalContent,
+ ModalCloseButton,
+ ModalFooter,
+ Stack,
+ Text,
+ Table,
+ Thead,
+ Tbody,
+ Tr,
+ Th,
+ Td,
+ TableContainer,
+ IconButton,
+ useToast,
+ useDisclosure,
+ Tag,
+ Alert,
+ AlertTitle,
+ AlertDescription,
+ Checkbox,
+ Textarea,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import { useForm } from "react-hook-form";
+import { TbCopy, TbPlus, TbTrash, TbPencil } from "react-icons/tb";
+import relativeTime from "dayjs/plugin/relativeTime";
+import { useAsyncFn } from "react-use";
+import API from "@/lib/api";
+import { ACCEPTABLE_WEBPAGE_TYPES, isSitemapUrl } from "@/lib/datasources";
+
+dayjs.extend(relativeTime);
+
+function DocumentRow({ id, name, createdAt, type, onDelete, onEdit }) {
+ const toast = useToast();
+ const copyToClipboard = (text) => {
+ navigator.clipboard.writeText(text);
+
+ toast({
+ description: "Copied to clipboard",
+ position: "top",
+ colorScheme: "gray",
+ });
+ };
+
+ const [{ loading: isDeleting }, handleDelete] = useAsyncFn(
+ async (id) => {
+ await onDelete(id);
+
+ toast({
+ description: "Document deleted",
+ position: "top",
+ colorScheme: "gray",
+ });
+ },
+ [onDelete]
+ );
+
+ return (
+
+ |
+
+ {name}
+
+ |
+
+
+ {type}
+
+ |
+
+
+ {id}
+ }
+ onClick={() => copyToClipboard(id)}
+ />
+
+ |
+
+ {dayjs(createdAt).fromNow()}
+ |
+
+
+ }
+ onClick={() => onEdit(id)}
+ />
+
+
+ ) : (
+
+ )
+ }
+ onClick={() => handleDelete(id)}
+ />
+
+ |
+
+ );
+}
+
+export default function Webpages({ data, session }) {
+ const [filteredData, setData] = useState();
+ const [isCreatingDocument, setIsCreatingDocument] = useState();
+ const { isOpen, onClose, onOpen } = useDisclosure();
+ const [selectedDocument, setSelectedDocument] = useState();
+ const router = useRouter();
+ const api = new API(session);
+ const toast = useToast();
+ const {
+ formState: { isSubmitting, errors },
+ handleSubmit,
+ register,
+ reset,
+ setValue,
+ watch,
+ } = useForm();
+ const url = watch("url");
+
+ const onCancel = async () => {
+ reset();
+ setSelectedDocument();
+ onClose();
+ };
+
+ const onUpdate = async (values) => {
+ const { name, description } = values;
+ const payload = {
+ name,
+ description,
+ };
+
+ await api.patchDocument(selectedDocument, payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Updated Document", { ...payload });
+ }
+
+ toast({
+ description: "Document updated",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ setData();
+ router.refresh();
+ reset();
+ setSelectedDocument();
+ onClose();
+ };
+
+ const handleSearch = ({ searchTerm }) => {
+ if (!searchTerm) {
+ setData(data);
+ }
+
+ const keysToFilter = ["name", "type"];
+ const filteredItems = data.filter((item) =>
+ keysToFilter.some((key) =>
+ item[key].toString().toLowerCase().includes(searchTerm.toLowerCase())
+ )
+ );
+
+ setData(filteredItems);
+ };
+
+ const handleDelete = async (id) => {
+ await api.deleteDocument({ id });
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Deleted Document", { id });
+ }
+
+ setData();
+ router.refresh();
+ };
+
+ const handleEdit = async (documentId) => {
+ const document = data.find(({ id }) => id === documentId);
+
+ setSelectedDocument(documentId);
+ setValue("name", document?.name);
+ setValue("description", document?.description);
+ setValue("url", document?.url);
+ onOpen();
+ };
+
+ const onSubmit = useCallback(async (values) => {
+ const { name, description, url, include_subpages, ...metadata } = values;
+ const type = isSitemapUrl(url) ? "SITEMAP" : "URL";
+ const payload = {
+ name: name,
+ description,
+ url,
+ type,
+ metadata,
+ };
+
+ await api.createDocument(payload);
+
+ if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
+ analytics.track("Created Webpage", { ...payload });
+ }
+
+ toast({
+ description: "Webpage created",
+ position: "top",
+ colorScheme: "gray",
+ });
+
+ onCancel();
+
+ router.refresh();
+ }, []);
+
+ return (
+
+
+
+ } onClick={onOpen}>
+ Add webpage
+
+
+ Add any webpage
+
+
+
+
+ handleSearch(values)}
+ onReset={() => setData(data)}
+ />
+
+
+
+
+ | Name |
+ Type |
+ ID |
+ Created at |
+ |
+
+
+
+ {filteredData
+ ? filteredData
+ ?.filter(({ type }) =>
+ ACCEPTABLE_WEBPAGE_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))
+ : data
+ ?.filter(({ type }) =>
+ ACCEPTABLE_WEBPAGE_TYPES.includes(type)
+ )
+ .map(({ id, name, createdAt, type, url }) => (
+ handleDelete(id)}
+ onEdit={(id) => handleEdit(id)}
+ />
+ ))}
+
+
+
+
+
+
+
+
+ {selectedDocument ? "Update webpage" : "Add webpage"}
+
+
+
+
+ {isSitemapUrl(url) && (
+
+
+
+ Looks like you are trying to import a Sitemap?
+
+
+
+ You can choose to filter out which urls you using the{" "}
+ Filter URLs field. This is recommended since
+ Sitemaps can be very large and take a long time to ingest.
+
+
+ )}
+
+
+ Name
+
+ The name of the webpage.
+ {errors?.name && (
+ Invalid name
+ )}
+
+
+ Description
+
+
+ What is this document useful for?
+
+ {errors?.description && (
+ Invalid description
+ )}
+
+
+ URL
+
+ Add the webpage URL
+ {errors?.url && (
+ Invalid URL
+ )}
+
+ {isSitemapUrl(url) && (
+
+ Filter URLs
+
+
+ Comma separated list of urls to ingest
+
+
+ )}
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/ui/app/datasources/client-page.js b/ui/app/datasources/client-page.js
new file mode 100644
index 000000000..df88b2f87
--- /dev/null
+++ b/ui/app/datasources/client-page.js
@@ -0,0 +1,61 @@
+"use client";
+import {
+ Heading,
+ HStack,
+ Stack,
+ Text,
+ Tabs,
+ TabList,
+ TabPanels,
+ Tab,
+ TabPanel,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import Files from "./_components/files";
+import Webpages from "./_components/webpages";
+import Applications from "./_components/applications";
+import relativeTime from "dayjs/plugin/relativeTime";
+
+dayjs.extend(relativeTime);
+
+export default function DatasourceClientPage({ data, session }) {
+ return (
+
+
+
+
+ Datasources
+
+
+ Upload files or connect third-party apps and use them in your
+ agents.
+
+
+
+
+
+ Files
+ Webpages
+ Applications
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/ui/app/documents/page.js b/ui/app/datasources/page.js
similarity index 77%
rename from ui/app/documents/page.js
rename to ui/app/datasources/page.js
index 8e6da36d1..157a6073e 100644
--- a/ui/app/documents/page.js
+++ b/ui/app/datasources/page.js
@@ -1,6 +1,6 @@
import { getServerSession } from "next-auth/next";
import { options } from "@/lib/next-auth";
-import DocumentsClientPage from "./client-page";
+import DatasourceClientPage from "./client-page";
import Api from "@/lib/api";
export const metadata = {
@@ -13,5 +13,5 @@ export default async function ApiTokens() {
const api = new Api(session);
const documents = await api.getDocuments();
- return ;
+ return ;
}
diff --git a/ui/app/documents/client-page.js b/ui/app/documents/client-page.js
deleted file mode 100644
index 676a97c74..000000000
--- a/ui/app/documents/client-page.js
+++ /dev/null
@@ -1,520 +0,0 @@
-"use client";
-import { useState } from "react";
-import {
- Alert,
- Button,
- FormControl,
- FormLabel,
- Heading,
- HStack,
- Icon,
- Input,
- Modal,
- ModalOverlay,
- ModalContent,
- ModalHeader,
- ModalFooter,
- ModalBody,
- ModalCloseButton,
- Stack,
- Select,
- Text,
- useDisclosure,
- FormHelperText,
- FormErrorMessage,
- IconButton,
- useToast,
- Box,
- Tag,
- SimpleGrid,
- Textarea,
- Center,
- Spinner,
-} from "@chakra-ui/react";
-import dayjs from "dayjs";
-import NextImage from "next/image";
-import { useRouter } from "next/navigation";
-import { TbPlus, TbCopy, TbTrash, TbPencil } from "react-icons/tb";
-import { useForm } from "react-hook-form";
-import API from "@/lib/api";
-import { analytics } from "@/lib/analytics";
-import { usePsychicLink } from "@psychic-api/link";
-import SearchBar from "../_components/search-bar";
-import relativeTime from "dayjs/plugin/relativeTime";
-
-dayjs.extend(relativeTime);
-
-function DocumentCard({ id, name, createdAt, type, url, onDelete, onEdit }) {
- const toast = useToast();
- const copyToClipboard = (text) => {
- navigator.clipboard.writeText(text);
-
- toast({
- description: "Copied to clipboard",
- position: "top",
- colorScheme: "gray",
- });
- };
-
- return (
-
-
-
- {name}
-
-
- {dayjs(createdAt).fromNow()}
-
-
-
- {`Id: ${id}`}
-
-
-
- {type}
-
-
- }
- onClick={() => onEdit(id)}
- />
- }
- onClick={() => copyToClipboard(id)}
- />
- }
- onClick={() => onDelete(id)}
- />
-
-
-
- );
-}
-
-export default function DocumentsClientPage({ data, session }) {
- const [filteredData, setData] = useState();
- const [isCreatingDocument, setIsCreatingDocument] = useState();
- const { isOpen, onClose, onOpen } = useDisclosure();
- const [selectedDocument, setSelectedDocument] = useState();
- const router = useRouter();
- const api = new API(session);
- const toast = useToast();
- const {
- formState: { isSubmitting, errors },
- handleSubmit,
- register,
- reset,
- setValue,
- watch,
- } = useForm();
-
- const documentType = watch("type");
- const url = watch("url");
- const { open, isReady, isLoading } = usePsychicLink(
- process.env.NEXT_PUBLIC_PSYCHIC_PUBLIC_KEY,
- async (newConnection) => {
- setIsCreatingDocument(true);
- await api.createDocument({
- name: `Psychic: ${newConnection.connectorId}`,
- type: "PSYCHIC",
- metadata: {
- connectorId: newConnection.connectorId,
- },
- });
-
- toast({
- description: "Psychic connection created!",
- position: "top",
- colorScheme: "gray",
- });
-
- setData();
- onClose();
- setIsCreatingDocument();
- router.refresh();
- }
- );
- const shouldShowPsychic = process.env.NEXT_PUBLIC_PSYCHIC_PUBLIC_KEY;
-
- const onCancel = async () => {
- reset();
- setSelectedDocument();
- onClose();
- };
-
- const onSubmit = async (values) => {
- const {
- type,
- name,
- description,
- url,
- auth_type,
- auth_key,
- auth_value,
- ...metadata
- } = values;
- const payload = {
- name,
- metadata,
- description,
- type,
- url,
- authorization: auth_key && {
- type: auth_type,
- key: auth_key,
- value: auth_value,
- },
- };
-
- if (selectedDocument) {
- await api.patchDocument(selectedDocument, payload);
-
- if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
- analytics.track("Updated Document", { ...payload });
- }
- } else {
- await api.createDocument(payload);
- if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
- analytics.track("Created Document", { ...payload });
- }
- }
-
- toast({
- description: selectedDocument ? "Document updated" : "Document created",
- position: "top",
- colorScheme: "gray",
- });
-
- setData();
- router.refresh();
- reset();
- setSelectedDocument();
- onClose();
- };
-
- const handleDelete = async (id) => {
- await api.deleteDocument({ id });
-
- if (process.env.NEXT_PUBLIC_SEGMENT_WRITE_KEY) {
- analytics.track("Deleted Document", { id });
- }
-
- toast({
- description: "Document deleted",
- position: "top",
- colorScheme: "gray",
- });
- setData();
- router.refresh();
- };
-
- const onConnectAPI = async () => {
- open(session.user.user?.id || session.user.id);
- };
-
- const handleEdit = async (documentId) => {
- const document = data.find(({ id }) => id === documentId);
-
- setSelectedDocument(documentId);
- setValue("name", document?.name);
- setValue("url", document?.url);
- setValue("type", document?.type);
- setValue("description", document?.description);
- onOpen();
- };
-
- const handleSearch = ({ searchTerm }) => {
- if (!searchTerm) {
- setData(data);
- }
-
- const keysToFilter = ["name", "type"];
- const filteredItems = data.filter((item) =>
- keysToFilter.some((key) =>
- item[key].toString().toLowerCase().includes(searchTerm.toLowerCase())
- )
- );
-
- setData(filteredItems);
- };
-
- return (
-
-
-
-
- Documents
-
-
- Upload documents and use them to do question answering.
-
-
- }
- alignSelf="flex-start"
- onClick={onOpen}
- >
- New document
-
-
- handleSearch(values)}
- onReset={() => setData(data)}
- />
-
-
- {filteredData
- ? filteredData?.map(({ id, name, createdAt, type, url }) => (
- handleDelete(id)}
- onEdit={(id) => handleEdit(id)}
- />
- ))
- : data?.map(({ id, name, createdAt, type, url }) => (
- handleDelete(id)}
- onEdit={(id) => handleEdit(id)}
- />
- ))}
-
-
-
-
-
-
- {selectedDocument ? "Update document" : "New document"}
-
-
-
- {isCreatingDocument && (
-
-
-
- Creating document...
-
-
- )}
- {!isCreatingDocument && (
-
- {shouldShowPsychic && (
-
-
-
-
-
- Psychic{" "}
-
- New
-
-
-
-
- Connect to Google Drive, Jira, Zendesk, Dropox etc.
-
-
-
-
-
- )}
-
-
- Name
-
- A document name.
- {errors?.name && (
- Invalid name
- )}
-
-
- Description
-
-
- What is this document useful for?
-
- {errors?.description && (
- Invalid description
- )}
-
- {documentType === "URL" ? (
-
- URL
-
-
- A comma separated list of urls.
-
- {errors?.url && (
- Invalid URL
- )}
-
- ) : (
-
- URL
-
-
- A publicly accessible URL to your document.
-
- {errors?.url && (
- Invalid URL
- )}
-
- )}
-
-
- Type
-
- {errors?.type && (
- Invalid type
- )}
-
- {documentType === "GITHUB_REPOSITORY" && (
-
-
- Branch
-
-
-
- )}
- {documentType === "OPENAPI" && (
-
-
- This feature is exeperimental, use with caution.
-
-
- Authorization
-
-
-
-
-
-
-
- If the OpenApi spec your are using requires
- authentication you need to use the fields above.
-
-
-
-
- )}
-
-
- )}
-
-
-
-
-
-
-
-
- );
-}
diff --git a/ui/lib/datasources.js b/ui/lib/datasources.js
new file mode 100644
index 000000000..f94bfb4c3
--- /dev/null
+++ b/ui/lib/datasources.js
@@ -0,0 +1,200 @@
+import AWS from "aws-sdk";
+
+AWS.config.update({
+ accessKeyId: process.env.NEXT_PUBLIC_AMAZON_S3_ACCESS_KEY_ID,
+ secretAccessKey: process.env.NEXT_PUBLIC_AMAZON_S3_SECRET_ACCESS_KEY,
+ region: process.env.NEXT_PUBLIC_AWS_S3_REGION,
+});
+
+const s3 = new AWS.S3();
+
+export const APPLICATIONS = [
+ {
+ id: "GITHUB_REPOSITORY",
+ name: "Github",
+ logo: "./github.png",
+ is_live: true,
+ inputs: [
+ {
+ key: "url",
+ name: "Repository URL",
+ placeholder: "https://github.com/homanp/superagent",
+ helpText: "The full URL to your Github repository",
+ type: "input",
+ required: true,
+ },
+ {
+ key: "branch",
+ name: "Branch",
+ placeholder: "main",
+ helpText: "Select the repo branch",
+ type: "input",
+ required: true,
+ },
+ ],
+ },
+ {
+ id: "AIRTABLE",
+ name: "Airtable",
+ logo: "./airtable.jpeg",
+ is_live: true,
+ inputs: [
+ {
+ key: "api_key",
+ name: "Personal access token",
+ placeholder: "",
+ helpText: "Enter your personal access token",
+ type: "input",
+ required: true,
+ },
+ {
+ key: "base_id",
+ name: "Base ID",
+ placeholder: "",
+ helpText: "Enter the base ID",
+ type: "input",
+ required: true,
+ },
+ {
+ key: "table_id",
+ name: "Table ID",
+ placeholder: "",
+ helpText: "Enter the table ID",
+ type: "input",
+ required: true,
+ },
+ ],
+ },
+ {
+ id: "STRIPE",
+ name: "Stripe",
+ logo: "./stripe.jpeg",
+ is_live: false,
+ inputs: [
+ {
+ key: "client_secret",
+ name: "Client secret",
+ placeholder: "Stripe client secret",
+ helpText: "Your stripe client secret",
+ type: "input",
+ required: true,
+ },
+ {
+ key: "account_id",
+ name: "Account ID",
+ placeholder: "Stripe account id",
+ helpText: "Your stripe account ID",
+ type: "input",
+ required: true,
+ },
+ {
+ key: "start_date",
+ name: "Start date",
+ type: "date",
+ helpText: "From what date do you want to fetch data",
+ required: true,
+ },
+ {
+ key: "stream",
+ name: "Resource",
+ type: "select",
+ required: true,
+ helpText: "Select a resource",
+ options: [
+ {
+ title: "Invoides",
+ value: "invoices",
+ },
+ ],
+ },
+ ],
+ },
+ {
+ id: "ZENDESK",
+ name: "Zendesk",
+ logo: "./zendesk.png",
+ is_live: false,
+ inputs: [],
+ },
+ {
+ id: "INTERCOM",
+ name: "Intercom",
+ logo: "./intercom.png",
+ is_live: false,
+ inputs: [],
+ },
+ {
+ id: "HUBSPOT",
+ name: "Hubspot",
+ logo: "./hubspot.png",
+ is_live: false,
+ inputs: [],
+ },
+ {
+ id: "SALESFORCE",
+ name: "Salesforce",
+ logo: "./salesforce.png",
+ is_live: false,
+ inputs: [],
+ },
+ {
+ id: "GONG",
+ name: "Gong",
+ logo: "./gong.jpeg",
+ is_live: false,
+ inputs: [],
+ },
+ {
+ id: "SHOPIFY",
+ name: "Shopify",
+ logo: "./shopify.jpeg",
+ is_live: false,
+ inputs: [],
+ },
+];
+
+export const ACCEPTABLE_APPLICATION_TYPES = APPLICATIONS.map(({ id }) => {
+ return id;
+});
+
+export const ACCEPTABLE_STATIC_FILE_TYPES = ["CSV", "PDF", "TXT", "MARKDOWN"];
+
+export const ACCEPTABLE_WEBPAGE_TYPES = ["WEBPAGE", "URL", "SITEMAP"];
+
+export const MIME_TO_TYPE = {
+ "text/csv": "CSV",
+ "application/pdf": "PDF",
+ "text/plain": "TXT",
+ "text/markdown": "MARKDOWN",
+ ".md": "MARKDOWN",
+};
+
+export const getFileType = (mimeType) => {
+ return MIME_TO_TYPE[mimeType] || "Unknown";
+};
+
+export const uploadFile = async (file) => {
+ const result = await s3
+ .upload({
+ Bucket: process.env.NEXT_PUBLIC_AWS_S3_BUCKET,
+ Key: file.name,
+ Body: file,
+ ContentType: file.type,
+ })
+ .promise();
+
+ return result;
+};
+
+export const isSitemapUrl = (url) => {
+ return url?.endsWith(".xml") || url?.includes("/sitemap");
+};
+
+export const isGithubUrl = (url) => {
+ try {
+ const parsedUrl = new URL(url);
+ return parsedUrl.hostname === "github.com";
+ } catch (error) {
+ return false; // Invalid URL
+ }
+};
diff --git a/ui/lib/sidebar-menu.js b/ui/lib/sidebar-menu.js
index 436657637..d67af978f 100644
--- a/ui/lib/sidebar-menu.js
+++ b/ui/lib/sidebar-menu.js
@@ -2,7 +2,7 @@ import {
TbKey,
TbBook,
TbLogout,
- TbFileText,
+ TbDatabasePlus,
TbBrain,
TbCreditCard,
TbCodePlus,
@@ -39,18 +39,18 @@ export const MAIN_MENU = [
path: "/tools",
icon: TbTool,
},
+ {
+ id: "datasources",
+ label: "Datasources",
+ path: "/datasources",
+ icon: TbDatabasePlus,
+ },
{
id: "prompts",
label: "Prompts",
path: "/prompts",
icon: TbCodePlus,
},
- {
- id: "documents",
- label: "Documents",
- path: "/documents",
- icon: TbFileText,
- },
{
id: "logs",
label: "Logs",
diff --git a/ui/package-lock.json b/ui/package-lock.json
index b90f73895..4332851c9 100644
--- a/ui/package-lock.json
+++ b/ui/package-lock.json
@@ -26,6 +26,7 @@
"@uiw/react-codemirror": "^4.19.16",
"@vercel/analytics": "^1.0.1",
"analytics": "^0.8.1",
+ "aws-sdk": "^2.1431.0",
"axios": "^1.4.0",
"dayjs": "^1.11.8",
"eslint-config-next": "13.4.2",
@@ -3181,6 +3182,34 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/aws-sdk": {
+ "version": "2.1431.0",
+ "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1431.0.tgz",
+ "integrity": "sha512-p6NGyI6+BgojiGn6uW2If6v7uxRPO5C+aGy/M+9/Rhdk8a5n7l0123v9ZUnEJgAy0tsNkazL2ifzV33nc0aGNA==",
+ "dependencies": {
+ "buffer": "4.9.2",
+ "events": "1.1.1",
+ "ieee754": "1.1.13",
+ "jmespath": "0.16.0",
+ "querystring": "0.2.0",
+ "sax": "1.2.1",
+ "url": "0.10.3",
+ "util": "^0.12.4",
+ "uuid": "8.0.0",
+ "xml2js": "0.5.0"
+ },
+ "engines": {
+ "node": ">= 10.0.0"
+ }
+ },
+ "node_modules/aws-sdk/node_modules/uuid": {
+ "version": "8.0.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz",
+ "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/axe-core": {
"version": "4.7.2",
"resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.2.tgz",
@@ -3244,6 +3273,25 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
+ "node_modules/base64-js": {
+ "version": "1.5.1",
+ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
"node_modules/big-integer": {
"version": "1.6.51",
"resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz",
@@ -3283,6 +3331,21 @@
"node": ">=8"
}
},
+ "node_modules/buffer": {
+ "version": "4.9.2",
+ "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
+ "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==",
+ "dependencies": {
+ "base64-js": "^1.0.2",
+ "ieee754": "^1.1.4",
+ "isarray": "^1.0.0"
+ }
+ },
+ "node_modules/buffer/node_modules/isarray": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+ "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
+ },
"node_modules/bundle-name": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz",
@@ -4308,6 +4371,14 @@
"node": ">=0.10.0"
}
},
+ "node_modules/events": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
+ "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==",
+ "engines": {
+ "node": ">=0.4.x"
+ }
+ },
"node_modules/execa": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/execa/-/execa-7.1.1.tgz",
@@ -4944,6 +5015,11 @@
"resolved": "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.4.tgz",
"integrity": "sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ=="
},
+ "node_modules/ieee754": {
+ "version": "1.1.13",
+ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
+ "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
+ },
"node_modules/ignore": {
"version": "5.2.4",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",
@@ -5046,6 +5122,21 @@
"url": "https://github.com/sponsors/wooorm"
}
},
+ "node_modules/is-arguments": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
+ "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
+ "dependencies": {
+ "call-bind": "^1.0.2",
+ "has-tostringtag": "^1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/is-array-buffer": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz",
@@ -5162,6 +5253,20 @@
"node": ">=0.10.0"
}
},
+ "node_modules/is-generator-function": {
+ "version": "1.0.10",
+ "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
+ "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
+ "dependencies": {
+ "has-tostringtag": "^1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/is-glob": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
@@ -5391,6 +5496,14 @@
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
},
+ "node_modules/jmespath": {
+ "version": "0.16.0",
+ "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz",
+ "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==",
+ "engines": {
+ "node": ">= 0.6.0"
+ }
+ },
"node_modules/join-component": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/join-component/-/join-component-1.1.0.tgz",
@@ -7036,6 +7149,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/querystring": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
+ "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==",
+ "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.",
+ "engines": {
+ "node": ">=0.4.x"
+ }
+ },
"node_modules/queue-microtask": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -7623,6 +7745,11 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/sax": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
+ "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA=="
+ },
"node_modules/scheduler": {
"version": "0.23.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
@@ -8335,6 +8462,20 @@
"punycode": "^2.1.0"
}
},
+ "node_modules/url": {
+ "version": "0.10.3",
+ "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
+ "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==",
+ "dependencies": {
+ "punycode": "1.3.2",
+ "querystring": "0.2.0"
+ }
+ },
+ "node_modules/url/node_modules/punycode": {
+ "version": "1.3.2",
+ "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
+ "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw=="
+ },
"node_modules/use-callback-ref": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.0.tgz",
@@ -8376,6 +8517,18 @@
}
}
},
+ "node_modules/util": {
+ "version": "0.12.5",
+ "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",
+ "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==",
+ "dependencies": {
+ "inherits": "^2.0.3",
+ "is-arguments": "^1.0.4",
+ "is-generator-function": "^1.0.7",
+ "is-typed-array": "^1.1.3",
+ "which-typed-array": "^1.1.2"
+ }
+ },
"node_modules/uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
@@ -8522,6 +8675,26 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
},
+ "node_modules/xml2js": {
+ "version": "0.5.0",
+ "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz",
+ "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==",
+ "dependencies": {
+ "sax": ">=0.6.0",
+ "xmlbuilder": "~11.0.0"
+ },
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/xmlbuilder": {
+ "version": "11.0.1",
+ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+ "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
+ "engines": {
+ "node": ">=4.0"
+ }
+ },
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
diff --git a/ui/package.json b/ui/package.json
index 8d7ceadc4..ebf10a172 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -26,6 +26,7 @@
"@uiw/react-codemirror": "^4.19.16",
"@vercel/analytics": "^1.0.1",
"analytics": "^0.8.1",
+ "aws-sdk": "^2.1431.0",
"axios": "^1.4.0",
"dayjs": "^1.11.8",
"eslint-config-next": "13.4.2",
diff --git a/ui/public/airtable.jpeg b/ui/public/airtable.jpeg
new file mode 100644
index 000000000..6b089236c
Binary files /dev/null and b/ui/public/airtable.jpeg differ
diff --git a/ui/public/github.png b/ui/public/github.png
new file mode 100644
index 000000000..c19378d84
Binary files /dev/null and b/ui/public/github.png differ
diff --git a/ui/public/gong.jpeg b/ui/public/gong.jpeg
new file mode 100644
index 000000000..2291e1b40
Binary files /dev/null and b/ui/public/gong.jpeg differ
diff --git a/ui/public/hubspot.png b/ui/public/hubspot.png
new file mode 100644
index 000000000..c015ef109
Binary files /dev/null and b/ui/public/hubspot.png differ
diff --git a/ui/public/intercom.png b/ui/public/intercom.png
new file mode 100644
index 000000000..699182bd5
Binary files /dev/null and b/ui/public/intercom.png differ
diff --git a/ui/public/salesforce.png b/ui/public/salesforce.png
new file mode 100644
index 000000000..0be53d82d
Binary files /dev/null and b/ui/public/salesforce.png differ
diff --git a/ui/public/shopify.jpeg b/ui/public/shopify.jpeg
new file mode 100644
index 000000000..ab2c3bcc3
Binary files /dev/null and b/ui/public/shopify.jpeg differ
diff --git a/ui/public/stripe.jpeg b/ui/public/stripe.jpeg
new file mode 100644
index 000000000..4fe07a2b2
Binary files /dev/null and b/ui/public/stripe.jpeg differ
diff --git a/ui/public/zendesk.png b/ui/public/zendesk.png
new file mode 100644
index 000000000..54341ea82
Binary files /dev/null and b/ui/public/zendesk.png differ