first commit

Author: tylerhutcherson

.dockerignore

@@ -0,0 +1,5 @@
+# ignore .git folders
+.git
+
+# ignore specific.json files
+arxiv-metadata-oai-snapshot.json

.gitignore

@@ -0,0 +1,4 @@
+arxiv-metadata-oai-snapshot.json
+*.pkl
+*.DS_STORE
+*.log

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+FROM node:17.2-alpine AS ReactImage
+
+WORKDIR /app/frontend
+
+ENV NODE_PATH=/app/frontend/node_modules
+ENV PATH=$PATH:/app/frontend/node_modules/.bin
+
+COPY ./frontend/package.json ./
+RUN yarn install --no-optional
+
+ADD ./frontend ./
+RUN yarn build
+
+
+FROM python:3.8-slim-buster AS ApiImage
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+RUN python3 -m pip install --upgrade pip setuptools wheel
+
+WORKDIR /app/
+COPY ./data/ ./data
+
+RUN mkdir -p /app/backend
+WORKDIR /app/backend
+
+COPY ./backend/ .
+RUN pip install -e .
+
+# add static react files to fastapi image
+COPY --from=ReactImage /app/frontend/build /app/backend/vecsim_app/templates/build
+
+WORKDIR /app/backend/vecsim_app
+
+CMD ["sh", "./entrypoint.sh"]

Dockerfile

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Sam Partee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE

@@ -0,0 +1,29 @@
+MAKEFLAGS += --no-print-directory
+
+# Do not remove this block. It is used by the 'help' rule when
+# constructing the help output.
+# help:
+# help: Redis arXiv Search App Makefile help
+# help:
+
+SHELL:=/bin/bash
+CONDA_ACTIVATE=source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate
+
+# help: help                      - display this makefile's help information
+.PHONY: help
+help:
+	@grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://'
+
+# help:
+# help: Conda Environment Setup
+# help: -------------
+
+# help: env                   - setup a Python conda env for this application
+.PHONY: env
+env:
+	@conda create -n arXiv python=3.9 -y
+	$(CONDA_ACTIVATE) arXiv
+	@cd backend/ && pip install -r requirements.txt
+
+# help:
+# help:

Makefile

@@ -0,0 +1,97 @@
+
+<div align="center">
+    <a href="https://github.com/spartee/redis-vector-search"><img src="https://github.com/RedisVentures/redis-arXiv-search/blob/master/app/vecsim_app/data/redis-logo.png?raw=true" width="30%"><img></a>
+    <br />
+    <br />
+<div display="inline-block">
+    <a href="https://github.com/RedisVentures/redis-arXiv-search"><b>Code</b></a>&nbsp;&nbsp;&nbsp;
+    <a href="https://redis.io/docs/stack/search/reference/vectors/"><b>Redis VSS Documentation</b></a>&nbsp;&nbsp;&nbsp;
+  </div>
+    <br />
+    <br />
+</div>
+
+# Redis arXiv Search Demo 
+
+https://www.kaggle.com/code/foolofatook/zero-shot-classification-with-huggingface-pipeline
+https://huggingface.co/allenai/scibert_scivocab_uncased
+
+
+This arXiv demo showcases the vector search similarity (VSS) capability within Redis Stack and Redis Enterprise.
+Through the RediSearch module, vector types and indexes can be added to Redis. This turns Redis into
+a highly performant vector database which can be used for all types of applications.
+
+
+
+## Application
+
+This app was built as a Single Page Application (SPA) with the following components:
+
+- **[Redis Stack](https://redis.io/docs/stack/)**: Vector database + JSON storage
+- **[FastAPI](https://fastapi.tiangolo.com/)** (Python 3.8)
+  - JWT authentication using [OAuth2 "password
+    flow"](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/) and
+    PyJWT
+- **[Pydantic](https://pydantic-docs.helpmanual.io/)** for schema and validation
+- **[React](https://reactjs.org/)** (with Typescript)
+- **[Redis OM](https://redis.io/docs/stack/get-started/tutorials/stack-python/)** for ORM
+- **[Docker Compose](https://docs.docker.com/compose/)** for development
+- **[MaterialUI](https://material-ui.com/)** for some UI elements
+- **[React-Bootstrap](https://react-bootstrap.github.io/)** for some UI elements
+- **[Huggingface Sentence Transformers](https://huggingface.co/sentence-transformers)** for vector embedding creation
+
+Some inspiration was taken from this [Cookiecutter project](https://github.com/Buuntu/fastapi-react)
+and turned into a SPA application instead of a separate front-end server approach.
+
+
+### Datasets
+
+The dataset was taken from the the following [Kaggle link](https://www.kaggle.com/Cornell-University/arxiv).
+
+Download and extract the zip file and place the json file in the `data/` directory.
+
+## Running Locally
+
+### Using pre-built containers
+
+The easiest option to run locally is to use the following docker-compose file to launch the
+prebuilt container hosted on GitHub.
+
+```yaml
+version: '3.7'
+services:
+
+  redis-vector-db:
+    image: redis/redis-stack:latest
+    ports:
+      - 6379:6379
+      - 8001:8001
+
+  backend:
+    image: ghcr.io/spartee/redis-vss-fashion:v0.2.0
+    environment:
+      DEPLOYMENT: "dev"
+      REDIS_DATA_URL: 'redis://redis-vector-db:6379'
+      REDIS_OM_URL: 'redis://redis-vector-db:6379'
+      REDIS_HOST: 'redis-vector-db'
+      REDIS_PORT: 6379
+    expose:
+      - "8888"
+    ports:
+      - "8888:8888"
+    depends_on:
+      - "redis-vector-db"
+
+```
+
+To launch, run the following
+- ``docker compose up`` in same directory as ``docker-compose.yml``
+- Navigate to ``0.0.0.0:8888`` in a browser
+
+### Building the containers
+
+More to come here
+
+### Running outside docker
+
+More to come here

README.md

@@ -0,0 +1,8 @@
+fastapi
+httpx
+requests
+uvicorn
+aiofiles
+redis-om
+pandas
+sentence-transformers

backend/requirements.txt

@@ -0,0 +1,44 @@
+[metadata]
+name = arXiv_vecsim
+version=0.0.1
+description = Workshop demo of Redis vector similarity over arXiv papers
+long_description = file: README.md
+long_description_content_type=text/markdown
+
+url = https://github.com/RedisVentures/redis-arXiv-search
+project_urls =
+    Source = https://github.com/RedisVentures/redis-arXiv-search
+
+author = Tyler Hutcherson
+author_email = tyler.hutcherson@redis.com
+contact = Tyler Hutcherson
+contact_email = tyler.hutcherson@redis.com
+license = MIT
+keywords = redis, vector similarity, ai, machine learning
+classifiers =
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+
+[options]
+packages = find:
+setup_requires =
+    setuptools>=39.2
+include_package_data = True
+python_requires = >=3.7
+install_requires =
+    fastapi
+    httpx
+    requests
+    uvicorn
+    aiofiles
+    redis-om
+    pandas
+    ipython
+    numpy
+    jinja2
+    passlib
+    bcrypt
+    python-multipart
+    pyjwt
+    sentence-transformers

backend/setup.cfg

@@ -0,0 +1,4 @@
+
+from setuptools import setup
+
+setup()

backend/setup.py

@@ -0,0 +1,79 @@
+import typing as t
+import redis.asyncio as redis
+
+from fastapi import APIRouter
+from vecsim_app import config
+from vecsim_app.schema import (
+    SimilarityRequest,
+    UserTextSimilarityRequest
+)
+from vecsim_app.models import Paper
+from vecsim_app.query import create_query
+
+
+# loaded once here at start up
+from sentence_transformers import SentenceTransformer
+TEXT_MODEL = SentenceTransformer('allenai/scibert_scivocab_uncased')
+
+paper_router = r = APIRouter()
+redis_client = redis.from_url(config.REDIS_URL)
+
+async def papers_from_results(results) -> list:
+    return [await Paper.get(p.paper_pk) for p in results.docs]
+
+@r.get("/", response_model=t.List[Paper],
+       name="paper:get_paper_samples",
+       operation_id="get_papers_samples")
+async def get_papers(limit: int = 20, skip: int = 0):
+    pks = await Paper.all_pks()
+    if pks:
+        # TODO figure out how to slice async_generator
+        papers = []
+        i = 0
+        async for pk in pks:
+            if i >= skip and i < skip + limit:
+                papers.append(await Paper.get(pk))
+            if len(papers) == limit:
+                break
+            i += 1
+        return papers
+    return []
+
+@r.post("/vectorsearch/text",
+       response_model=t.List[Paper],
+       name="paper:find_similar_by_text",
+       operation_id="compute_text_similarity")
+async def find_papers_by_text(similarity_request: SimilarityRequest) -> t.List[Paper]:
+    q = create_query(
+        similarity_request.search_type,
+        similarity_request.number_of_results
+    )
+
+    # find the vector of the Paper listed in the request
+    paper_vector_key = "paper_vector:" + str(similarity_request.paper_id)
+    vector = await redis_client.hget(paper_vector_key, "vector")
+
+    # obtain results of the query
+    results = await redis_client.ft().search(q, query_params={"vec_param": vector})
+
+    # Get Paper records of those results
+    return await papers_from_results(results)
+
+
+@r.post("/vectorsearch/text/user",
+       response_model=t.List[Paper],
+       name="paper:find_similar_by_user_text",
+       operation_id="compute_user_text_similarity")
+async def find_papers_by_user_text(similarity_request: UserTextSimilarityRequest) -> t.List[Paper]:
+    q = create_query(
+        similarity_request.search_type,
+        similarity_request.number_of_results
+    )
+
+    vector = TEXT_MODEL.encode(similarity_request.user_text)
+
+    # obtain results of the query
+    results = await redis_client.ft().search(q, query_params={"vec_param": vector.tobytes()})
+
+    # Get Paper records of those results
+    return await papers_from_results(results)

backend/vecsim_app/__init__.py

@@ -0,0 +1,15 @@
+import os
+
+PROJECT_NAME = "vecsim_app"
+API_DOCS = "/api/docs"
+OPENAPI_DOCS = "/api/openapi.json"
+INDEX_TYPE = os.environ.get("VECSIM_INDEX_TYPE", "HNSW")
+REDIS_HOST = os.environ.get("REDIS_HOST", "redis-vector-db")
+REDIS_PORT = os.environ.get("REDIS_PORT", 6379)
+REDIS_DB = os.environ.get("REDIS_DB", 0)
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "testing123")
+REDIS_URL = f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
+os.environ["REDIS_DATA_URL"] = REDIS_URL
+os.environ["REDIS_OM_URL"] = REDIS_URL
+API_V1_STR = "/api/v1"
+DATA_LOCATION = os.environ.get("DATA_LOCATION", "../../data")

backend/vecsim_app/api/__init__.py

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+python load_data.py
+
+python main.py