From 56471c8958ac1e5db50a51ee8a4c6f0780132b5a Mon Sep 17 00:00:00 2001 From: freakeinstein Date: Sun, 14 Jul 2019 18:39:30 +0530 Subject: [PATCH 1/3] update develop branch with master --- Dockerfile | 75 +++--- README.md | 16 +- src/init_aquila_db.sh | 5 +- src/test/vecdb_pb2.py | 532 +++++++++++++++++++++++++++++++++++++ src/test/vecdb_pb2_grpc.py | 97 +++++++ 5 files changed, 686 insertions(+), 39 deletions(-) create mode 100644 src/test/vecdb_pb2.py create mode 100644 src/test/vecdb_pb2_grpc.py diff --git a/Dockerfile b/Dockerfile index 6f8b9f0..996ebdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,39 +2,46 @@ FROM ubuntu:18.04 MAINTAINER a_mma # install couchdb -RUN apt-get update; apt-get install -y curl; apt-get install -y git -RUN echo "deb https://apache.bintray.com/couchdb-deb bionic main" \ - | tee -a /etc/apt/sources.list -RUN apt-get install -y gnupg -RUN curl -L https://couchdb.apache.org/repo/bintray-pubkey.asc \ - | apt-key add - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y couchdb - -# clone aquiladb src -RUN mkdir aquiladb && cd aquiladb && git clone https://github.com/a-mma/AquilaDB.git . - -# setup node environment -RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - -RUN apt install -y nodejs -RUN apt-get install -y make -RUN cd aquiladb/src && rm package-lock.json || true && npm install - +RUN apt-get update; apt-get install -y curl; apt-get install -y --no-install-recommends git; \ + echo "deb https://apache.bintray.com/couchdb-deb bionic main" \ + | tee -a /etc/apt/sources.list; \ + apt-get install -y --no-install-recommends gnupg; \ + curl -L https://couchdb.apache.org/repo/bintray-pubkey.asc \ + | apt-key add -; \ + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends couchdb; \ + apt-get purge -y --auto-remove gnupg; \ +# +# clone repository + git clone https://github.com/a-mma/AquilaDB.git; \ +# +# setup node environment + curl -sL https://deb.nodesource.com/setup_10.x | bash -; \ + apt install -y --no-install-recommends nodejs; \ + apt-get install -y --no-install-recommends make; \ + cd /AquilaDB/src && rm package-lock.json || true && npm install; \ + npm install pm2 -g; \ + npm cache clean --force; \ + apt-get purge -y --auto-remove curl make git; \ +# # setup python environment -RUN python3 --version -RUN apt-get -y install python3-pip -RUN apt-get install -y wget -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc -ENV PATH /opt/conda/bin:$PATH -RUN conda create -n myenv python && conda install faiss-cpu -c pytorch -RUN python -m pip install grpcio-tools - -# install pm2 to run services -RUN npm install pm2 -g + python3 --version; \ + apt-get -y install python3-pip wget --no-install-recommends; \ + wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc; \ + export PATH=/opt/conda/bin:$PATH; \ + conda config --set auto_update_conda False ; \ + conda create -n myenv python && conda install faiss-cpu=1.5.1 -c pytorch -y; \ + python -m pip install grpcio-tools; \ + apt-get purge -y --auto-remove wget; \ + conda clean --all; \ + rm -r "/opt/conda/pkgs/"; \ + rm -rf /var/lib/apt/lists/* \ +# +# make init script executable + chmod +x /AquilaDB/src/init_aquila_db.sh -RUN chmod +x /aquiladb/src/init_aquila_db.sh -CMD /aquiladb/src/init_aquila_db.sh && tail -f /dev/null \ No newline at end of file +CMD /AquilaDB/src/init_aquila_db.sh && tail -f /dev/null diff --git a/README.md b/README.md index ab49544..c803ed9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,17 @@ +![version](https://img.shields.io/github/release-pre/a-mma/AquilaDB.svg) ![license GPL 0.3](https://img.shields.io/github/license/a-mma/AquilaDB.svg) + +![docker build](https://img.shields.io/docker/cloud/automated/ammaorg/aquiladb.svg) ![docker build ststus](https://img.shields.io/docker/cloud/build/ammaorg/aquiladb.svg) ![docker stars](https://img.shields.io/docker/stars/ammaorg/aquiladb.svg) ![docker pulls](https://img.shields.io/docker/pulls/ammaorg/aquiladb.svg) + +![github open issues](https://img.shields.io/github/issues-raw/a-mma/AquilaDB.svg) ![github closed issues](https://img.shields.io/github/issues-closed-raw/a-mma/AquilaDB.svg) ![github stars](https://img.shields.io/github/stars/a-mma/AquilaDB.svg?style=social) + # AquilaDB **AquilaDB** is a **Resillient**, **Replicated**, **Decentralized**, **Host neutral** storage for **Feature Vectors** along with **Document Metadata**. Do **k-NN** retrieval from anywhere, even from the darkest rifts of Aquila (in progress). It is easy to setup and scales as the universe expands. -[https://github.com/a-mma/AquilaDB](https://github.com/a-mma/AquilaDB) +Github: [https://github.com/a-mma/AquilaDB](https://github.com/a-mma/AquilaDB) + +Docker Hub: [https://hub.docker.com/r/ammaorg/aquiladb](https://hub.docker.com/r/ammaorg/aquiladb) + +Documentation (dedicated Wiki page): [https://github.com/a-mma/AquilaDB/wiki](https://github.com/a-mma/AquilaDB/wiki) ![constellation](http://astronomyonline.org/Observation/Images/Constellations/ConstellationBig/Aquila.gif) @@ -9,13 +19,13 @@ Make sure your data is always available anywhere through any network. It is not necessory to be always online. Work offline, sync later. #### Replicated -Your data is replicated over nodes to attain eventual consistancy. +Your data is replicated over nodes to attain eventual consistency. #### Decentralized There is no single point of failure. #### Host Neutral -Want to use AWS, Azure, G-cloud or whatever? Or even a legion of laptops? Connect them together? No worries as long as they can talk each other. +Want to use AWS, Azure, G-cloud or whatever? Got a legion of laptops? Connect them together? No worries as long as they can talk each other. # Who is this for * If you are working on a data science project and need to store a hell lot of data and retrieve similar data based on some feature vector, this will be a useful tool to you, with extra benefits a real world web application needs. diff --git a/src/init_aquila_db.sh b/src/init_aquila_db.sh index 99d80fc..f567e09 100755 --- a/src/init_aquila_db.sh +++ b/src/init_aquila_db.sh @@ -7,7 +7,8 @@ service couchdb start sleep 5 # start micro services -cd /aquiladb/src/ +# with PM2 process manager +cd /AquilaDB/src/ pm2 start vecdb.js pm2 start peer_manager.js -pm2 start vecstore.py \ No newline at end of file +pm2 start vecstore.py diff --git a/src/test/vecdb_pb2.py b/src/test/vecdb_pb2.py new file mode 100644 index 0000000..6688b1e --- /dev/null +++ b/src/test/vecdb_pb2.py @@ -0,0 +1,532 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vecdb.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='vecdb.proto', + package='vecdb', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x0bvecdb.proto\x12\x05vecdb\"\x17\n\nvectorType\x12\t\n\x01\x65\x18\x01 \x03(\x02\"P\n\rsingleDocType\x12\x0b\n\x03_id\x18\x01 \x01(\t\x12!\n\x06vector\x18\x02 \x01(\x0b\x32\x11.vecdb.vectorType\x12\x0f\n\x07\x62\x36\x34\x64\x61ta\x18\x03 \x01(\x0c\"8\n\raddDocRequest\x12\'\n\tdocuments\x18\x01 \x03(\x0b\x32\x14.vecdb.singleDocType\"-\n\x0e\x61\x64\x64\x44ocResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x12\x0b\n\x03_id\x18\x02 \x03(\t\";\n\x10\x64\x65leteDocRequest\x12\'\n\tdocuments\x18\x01 \x03(\x0b\x32\x14.vecdb.singleDocType\"0\n\x11\x64\x65leteDocResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x12\x0b\n\x03_id\x18\x02 \x03(\t\"\x1f\n\x0e\x61\x64\x64NodeRequest\x12\r\n\x05peers\x18\x01 \x03(\t\"0\n\x0f\x61\x64\x64NodeResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x12\r\n\x05peers\x18\x02 \x03(\t\"A\n\x11getNearestRequest\x12!\n\x06matrix\x18\x01 \x03(\x0b\x32\x11.vecdb.vectorType\x12\t\n\x01k\x18\x02 \x01(\x05\"L\n\x12getNearestResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x12\x13\n\x0b\x64ist_matrix\x18\x02 \x01(\x0c\x12\x11\n\tdocuments\x18\x03 \x01(\x0c\x32\x8e\x02\n\x0cVecdbService\x12;\n\x0c\x61\x64\x64\x44ocuments\x12\x14.vecdb.addDocRequest\x1a\x15.vecdb.addDocResponse\x12\x44\n\x0f\x64\x65leteDocuments\x12\x17.vecdb.deleteDocRequest\x1a\x18.vecdb.deleteDocResponse\x12\x38\n\x07\x61\x64\x64Node\x12\x15.vecdb.addNodeRequest\x1a\x16.vecdb.addNodeResponse\x12\x41\n\ngetNearest\x12\x18.vecdb.getNearestRequest\x1a\x19.vecdb.getNearestResponseb\x06proto3') +) + + + + +_VECTORTYPE = _descriptor.Descriptor( + name='vectorType', + full_name='vecdb.vectorType', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='e', full_name='vecdb.vectorType.e', index=0, + number=1, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=22, + serialized_end=45, +) + + +_SINGLEDOCTYPE = _descriptor.Descriptor( + name='singleDocType', + full_name='vecdb.singleDocType', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='_id', full_name='vecdb.singleDocType._id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='vector', full_name='vecdb.singleDocType.vector', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='b64data', full_name='vecdb.singleDocType.b64data', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=47, + serialized_end=127, +) + + +_ADDDOCREQUEST = _descriptor.Descriptor( + name='addDocRequest', + full_name='vecdb.addDocRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='documents', full_name='vecdb.addDocRequest.documents', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=129, + serialized_end=185, +) + + +_ADDDOCRESPONSE = _descriptor.Descriptor( + name='addDocResponse', + full_name='vecdb.addDocResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='status', full_name='vecdb.addDocResponse.status', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='_id', full_name='vecdb.addDocResponse._id', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=187, + serialized_end=232, +) + + +_DELETEDOCREQUEST = _descriptor.Descriptor( + name='deleteDocRequest', + full_name='vecdb.deleteDocRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='documents', full_name='vecdb.deleteDocRequest.documents', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=234, + serialized_end=293, +) + + +_DELETEDOCRESPONSE = _descriptor.Descriptor( + name='deleteDocResponse', + full_name='vecdb.deleteDocResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='status', full_name='vecdb.deleteDocResponse.status', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='_id', full_name='vecdb.deleteDocResponse._id', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=295, + serialized_end=343, +) + + +_ADDNODEREQUEST = _descriptor.Descriptor( + name='addNodeRequest', + full_name='vecdb.addNodeRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='peers', full_name='vecdb.addNodeRequest.peers', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=345, + serialized_end=376, +) + + +_ADDNODERESPONSE = _descriptor.Descriptor( + name='addNodeResponse', + full_name='vecdb.addNodeResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='status', full_name='vecdb.addNodeResponse.status', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='peers', full_name='vecdb.addNodeResponse.peers', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=378, + serialized_end=426, +) + + +_GETNEARESTREQUEST = _descriptor.Descriptor( + name='getNearestRequest', + full_name='vecdb.getNearestRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='matrix', full_name='vecdb.getNearestRequest.matrix', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='k', full_name='vecdb.getNearestRequest.k', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=428, + serialized_end=493, +) + + +_GETNEARESTRESPONSE = _descriptor.Descriptor( + name='getNearestResponse', + full_name='vecdb.getNearestResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='status', full_name='vecdb.getNearestResponse.status', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dist_matrix', full_name='vecdb.getNearestResponse.dist_matrix', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='documents', full_name='vecdb.getNearestResponse.documents', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=495, + serialized_end=571, +) + +_SINGLEDOCTYPE.fields_by_name['vector'].message_type = _VECTORTYPE +_ADDDOCREQUEST.fields_by_name['documents'].message_type = _SINGLEDOCTYPE +_DELETEDOCREQUEST.fields_by_name['documents'].message_type = _SINGLEDOCTYPE +_GETNEARESTREQUEST.fields_by_name['matrix'].message_type = _VECTORTYPE +DESCRIPTOR.message_types_by_name['vectorType'] = _VECTORTYPE +DESCRIPTOR.message_types_by_name['singleDocType'] = _SINGLEDOCTYPE +DESCRIPTOR.message_types_by_name['addDocRequest'] = _ADDDOCREQUEST +DESCRIPTOR.message_types_by_name['addDocResponse'] = _ADDDOCRESPONSE +DESCRIPTOR.message_types_by_name['deleteDocRequest'] = _DELETEDOCREQUEST +DESCRIPTOR.message_types_by_name['deleteDocResponse'] = _DELETEDOCRESPONSE +DESCRIPTOR.message_types_by_name['addNodeRequest'] = _ADDNODEREQUEST +DESCRIPTOR.message_types_by_name['addNodeResponse'] = _ADDNODERESPONSE +DESCRIPTOR.message_types_by_name['getNearestRequest'] = _GETNEARESTREQUEST +DESCRIPTOR.message_types_by_name['getNearestResponse'] = _GETNEARESTRESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +vectorType = _reflection.GeneratedProtocolMessageType('vectorType', (_message.Message,), dict( + DESCRIPTOR = _VECTORTYPE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.vectorType) + )) +_sym_db.RegisterMessage(vectorType) + +singleDocType = _reflection.GeneratedProtocolMessageType('singleDocType', (_message.Message,), dict( + DESCRIPTOR = _SINGLEDOCTYPE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.singleDocType) + )) +_sym_db.RegisterMessage(singleDocType) + +addDocRequest = _reflection.GeneratedProtocolMessageType('addDocRequest', (_message.Message,), dict( + DESCRIPTOR = _ADDDOCREQUEST, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.addDocRequest) + )) +_sym_db.RegisterMessage(addDocRequest) + +addDocResponse = _reflection.GeneratedProtocolMessageType('addDocResponse', (_message.Message,), dict( + DESCRIPTOR = _ADDDOCRESPONSE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.addDocResponse) + )) +_sym_db.RegisterMessage(addDocResponse) + +deleteDocRequest = _reflection.GeneratedProtocolMessageType('deleteDocRequest', (_message.Message,), dict( + DESCRIPTOR = _DELETEDOCREQUEST, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.deleteDocRequest) + )) +_sym_db.RegisterMessage(deleteDocRequest) + +deleteDocResponse = _reflection.GeneratedProtocolMessageType('deleteDocResponse', (_message.Message,), dict( + DESCRIPTOR = _DELETEDOCRESPONSE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.deleteDocResponse) + )) +_sym_db.RegisterMessage(deleteDocResponse) + +addNodeRequest = _reflection.GeneratedProtocolMessageType('addNodeRequest', (_message.Message,), dict( + DESCRIPTOR = _ADDNODEREQUEST, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.addNodeRequest) + )) +_sym_db.RegisterMessage(addNodeRequest) + +addNodeResponse = _reflection.GeneratedProtocolMessageType('addNodeResponse', (_message.Message,), dict( + DESCRIPTOR = _ADDNODERESPONSE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.addNodeResponse) + )) +_sym_db.RegisterMessage(addNodeResponse) + +getNearestRequest = _reflection.GeneratedProtocolMessageType('getNearestRequest', (_message.Message,), dict( + DESCRIPTOR = _GETNEARESTREQUEST, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.getNearestRequest) + )) +_sym_db.RegisterMessage(getNearestRequest) + +getNearestResponse = _reflection.GeneratedProtocolMessageType('getNearestResponse', (_message.Message,), dict( + DESCRIPTOR = _GETNEARESTRESPONSE, + __module__ = 'vecdb_pb2' + # @@protoc_insertion_point(class_scope:vecdb.getNearestResponse) + )) +_sym_db.RegisterMessage(getNearestResponse) + + + +_VECDBSERVICE = _descriptor.ServiceDescriptor( + name='VecdbService', + full_name='vecdb.VecdbService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=574, + serialized_end=844, + methods=[ + _descriptor.MethodDescriptor( + name='addDocuments', + full_name='vecdb.VecdbService.addDocuments', + index=0, + containing_service=None, + input_type=_ADDDOCREQUEST, + output_type=_ADDDOCRESPONSE, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='deleteDocuments', + full_name='vecdb.VecdbService.deleteDocuments', + index=1, + containing_service=None, + input_type=_DELETEDOCREQUEST, + output_type=_DELETEDOCRESPONSE, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='addNode', + full_name='vecdb.VecdbService.addNode', + index=2, + containing_service=None, + input_type=_ADDNODEREQUEST, + output_type=_ADDNODERESPONSE, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='getNearest', + full_name='vecdb.VecdbService.getNearest', + index=3, + containing_service=None, + input_type=_GETNEARESTREQUEST, + output_type=_GETNEARESTRESPONSE, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_VECDBSERVICE) + +DESCRIPTOR.services_by_name['VecdbService'] = _VECDBSERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/src/test/vecdb_pb2_grpc.py b/src/test/vecdb_pb2_grpc.py new file mode 100644 index 0000000..d377bc1 --- /dev/null +++ b/src/test/vecdb_pb2_grpc.py @@ -0,0 +1,97 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import vecdb_pb2 as vecdb__pb2 + + +class VecdbServiceStub(object): + # missing associated documentation comment in .proto file + pass + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.addDocuments = channel.unary_unary( + '/vecdb.VecdbService/addDocuments', + request_serializer=vecdb__pb2.addDocRequest.SerializeToString, + response_deserializer=vecdb__pb2.addDocResponse.FromString, + ) + self.deleteDocuments = channel.unary_unary( + '/vecdb.VecdbService/deleteDocuments', + request_serializer=vecdb__pb2.deleteDocRequest.SerializeToString, + response_deserializer=vecdb__pb2.deleteDocResponse.FromString, + ) + self.addNode = channel.unary_unary( + '/vecdb.VecdbService/addNode', + request_serializer=vecdb__pb2.addNodeRequest.SerializeToString, + response_deserializer=vecdb__pb2.addNodeResponse.FromString, + ) + self.getNearest = channel.unary_unary( + '/vecdb.VecdbService/getNearest', + request_serializer=vecdb__pb2.getNearestRequest.SerializeToString, + response_deserializer=vecdb__pb2.getNearestResponse.FromString, + ) + + +class VecdbServiceServicer(object): + # missing associated documentation comment in .proto file + pass + + def addDocuments(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def deleteDocuments(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def addNode(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getNearest(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_VecdbServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'addDocuments': grpc.unary_unary_rpc_method_handler( + servicer.addDocuments, + request_deserializer=vecdb__pb2.addDocRequest.FromString, + response_serializer=vecdb__pb2.addDocResponse.SerializeToString, + ), + 'deleteDocuments': grpc.unary_unary_rpc_method_handler( + servicer.deleteDocuments, + request_deserializer=vecdb__pb2.deleteDocRequest.FromString, + response_serializer=vecdb__pb2.deleteDocResponse.SerializeToString, + ), + 'addNode': grpc.unary_unary_rpc_method_handler( + servicer.addNode, + request_deserializer=vecdb__pb2.addNodeRequest.FromString, + response_serializer=vecdb__pb2.addNodeResponse.SerializeToString, + ), + 'getNearest': grpc.unary_unary_rpc_method_handler( + servicer.getNearest, + request_deserializer=vecdb__pb2.getNearestRequest.FromString, + response_serializer=vecdb__pb2.getNearestResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'vecdb.VecdbService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) From 35e5758cf6f1a306a49fe68656923f7b545ec68b Mon Sep 17 00:00:00 2001 From: freakeinstein Date: Sun, 14 Jul 2019 18:53:50 +0530 Subject: [PATCH 2/3] match Dockerfile to master branch --- Dockerfile | 75 +++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index 996ebdc..6f8b9f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,46 +2,39 @@ FROM ubuntu:18.04 MAINTAINER a_mma # install couchdb -RUN apt-get update; apt-get install -y curl; apt-get install -y --no-install-recommends git; \ - echo "deb https://apache.bintray.com/couchdb-deb bionic main" \ - | tee -a /etc/apt/sources.list; \ - apt-get install -y --no-install-recommends gnupg; \ - curl -L https://couchdb.apache.org/repo/bintray-pubkey.asc \ - | apt-key add -; \ - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends couchdb; \ - apt-get purge -y --auto-remove gnupg; \ -# -# clone repository - git clone https://github.com/a-mma/AquilaDB.git; \ -# -# setup node environment - curl -sL https://deb.nodesource.com/setup_10.x | bash -; \ - apt install -y --no-install-recommends nodejs; \ - apt-get install -y --no-install-recommends make; \ - cd /AquilaDB/src && rm package-lock.json || true && npm install; \ - npm install pm2 -g; \ - npm cache clean --force; \ - apt-get purge -y --auto-remove curl make git; \ -# +RUN apt-get update; apt-get install -y curl; apt-get install -y git +RUN echo "deb https://apache.bintray.com/couchdb-deb bionic main" \ + | tee -a /etc/apt/sources.list +RUN apt-get install -y gnupg +RUN curl -L https://couchdb.apache.org/repo/bintray-pubkey.asc \ + | apt-key add - +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y couchdb + +# clone aquiladb src +RUN mkdir aquiladb && cd aquiladb && git clone https://github.com/a-mma/AquilaDB.git . + +# setup node environment +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt install -y nodejs +RUN apt-get install -y make +RUN cd aquiladb/src && rm package-lock.json || true && npm install + # setup python environment - python3 --version; \ - apt-get -y install python3-pip wget --no-install-recommends; \ - wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc; \ - export PATH=/opt/conda/bin:$PATH; \ - conda config --set auto_update_conda False ; \ - conda create -n myenv python && conda install faiss-cpu=1.5.1 -c pytorch -y; \ - python -m pip install grpcio-tools; \ - apt-get purge -y --auto-remove wget; \ - conda clean --all; \ - rm -r "/opt/conda/pkgs/"; \ - rm -rf /var/lib/apt/lists/* \ -# -# make init script executable - chmod +x /AquilaDB/src/init_aquila_db.sh +RUN python3 --version +RUN apt-get -y install python3-pip +RUN apt-get install -y wget +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc +ENV PATH /opt/conda/bin:$PATH +RUN conda create -n myenv python && conda install faiss-cpu -c pytorch +RUN python -m pip install grpcio-tools + +# install pm2 to run services +RUN npm install pm2 -g -CMD /AquilaDB/src/init_aquila_db.sh && tail -f /dev/null +RUN chmod +x /aquiladb/src/init_aquila_db.sh +CMD /aquiladb/src/init_aquila_db.sh && tail -f /dev/null \ No newline at end of file From 414e1d959641c2e652b22e582429fddbcef650f9 Mon Sep 17 00:00:00 2001 From: freakeinstein Date: Sun, 14 Jul 2019 20:07:59 +0530 Subject: [PATCH 3/3] tested to be working Dockerfile --- Dockerfile | 12 ++++++------ src/init_aquila_db.sh | 2 +- src/test/client.js | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6f8b9f0..2c7d1f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,14 +10,14 @@ RUN curl -L https://couchdb.apache.org/repo/bintray-pubkey.asc \ | apt-key add - RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y couchdb -# clone aquiladb src -RUN mkdir aquiladb && cd aquiladb && git clone https://github.com/a-mma/AquilaDB.git . +# clone AquilaDB src +RUN mkdir AquilaDB && cd AquilaDB && git clone https://github.com/a-mma/AquilaDB.git . # setup node environment RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - RUN apt install -y nodejs RUN apt-get install -y make -RUN cd aquiladb/src && rm package-lock.json || true && npm install +RUN cd AquilaDB/src && rm package-lock.json || true && npm install # setup python environment RUN python3 --version @@ -30,11 +30,11 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc ENV PATH /opt/conda/bin:$PATH -RUN conda create -n myenv python && conda install faiss-cpu -c pytorch +RUN conda create -n myenv python && conda install faiss-cpu=1.5.1 -c pytorch -y RUN python -m pip install grpcio-tools # install pm2 to run services RUN npm install pm2 -g -RUN chmod +x /aquiladb/src/init_aquila_db.sh -CMD /aquiladb/src/init_aquila_db.sh && tail -f /dev/null \ No newline at end of file +RUN chmod +x /AquilaDB/src/init_aquila_db.sh +CMD /AquilaDB/src/init_aquila_db.sh && tail -f /dev/null \ No newline at end of file diff --git a/src/init_aquila_db.sh b/src/init_aquila_db.sh index f567e09..81eb3fc 100755 --- a/src/init_aquila_db.sh +++ b/src/init_aquila_db.sh @@ -11,4 +11,4 @@ sleep 5 cd /AquilaDB/src/ pm2 start vecdb.js pm2 start peer_manager.js -pm2 start vecstore.py +pm2 start vecstore.py \ No newline at end of file diff --git a/src/test/client.js b/src/test/client.js index 48a0722..61b98b2 100644 --- a/src/test/client.js +++ b/src/test/client.js @@ -4,9 +4,9 @@ const btoa = require('btoa') const atob = require('atob') const njs = require('numjs') -var itercnt = 201 -var rawcnt = 50 -var veclen = 785 +var itercnt = 1 +var rawcnt = 1 +var veclen = 8 var PROTO_PATH = __dirname + '/../proto/vecdb.proto'