BatsResearch · dotpyu · Sep 13, 2023 · Sep 13, 2023 · Sep 13, 2023
diff --git a/alfred/fm/utils.py b/alfred/fm/utils.py
@@ -277,6 +277,8 @@ def __init__(
         self.max_batch_size = max_batch_size
         self.ranked = False
         self.tokenizer = tokenizer
+        if self.tokenizer:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
         self.max_token_length = max_token_length
 
         if isinstance(self.queries[0], RankedQuery):

diff --git a/docs/alfred/client/cache/cache.md b/docs/alfred/client/cache/cache.md
@@ -372,4 +372,6 @@ Type: *str*
 ```python
 def to_metadata_string(**kwargs: Any) -> str:
     ...
-```
+```
+
+
diff --git a/docs/alfred/client/cache/dummy.md b/docs/alfred/client/cache/dummy.md
@@ -195,4 +195,6 @@ Write a prompt-response pair to the cache
 ```python
 def write(self, prompt: str, response: str, metadata: Optional[str] = None):
     ...
-```
+```
+
+
diff --git a/docs/alfred/client/cache/sqlite.md b/docs/alfred/client/cache/sqlite.md
@@ -328,4 +328,6 @@ def write_batch(
     self, prompts: List[str], responses: List[str], metadata: Optional[str] = None
 ):
     ...
-```
+```
+
+
diff --git a/docs/alfred/client/client.md b/docs/alfred/client/client.md
@@ -275,4 +275,6 @@ def score(
     self, query: Union[RankedQuery, Dict, List[RankedQuery], List[str]], **kwargs: Any
 ) -> Union[Response, List[Response]]:
     ...
-```
+```
+
+
diff --git a/docs/alfred/client/ssh/sshtunnel.md b/docs/alfred/client/ssh/sshtunnel.md
@@ -79,4 +79,6 @@ Stop the tunnel
 ```python
 def stop(self):
     ...
-```
+```
+
+
diff --git a/docs/alfred/client/ssh/utils.md b/docs/alfred/client/ssh/utils.md
@@ -94,4 +94,6 @@ Finds the next available port if given port is not available
 ```python
 def port_finder(port: Union[str, int], host: str = "") -> int:
     ...
-```
+```
+
+
diff --git a/docs/alfred/data/arrow.md b/docs/alfred/data/arrow.md
@@ -614,4 +614,6 @@ returns the version of the dataset
 ```python
 def version(self) -> str:
     ...
-```
+```
+
+
diff --git a/docs/alfred/data/dataset.md b/docs/alfred/data/dataset.md
@@ -194,4 +194,6 @@ returns the version of the dataset
 @property
 def version(self) -> str:
     ...
-```
+```
+
+
diff --git a/docs/alfred/data/wrench.md b/docs/alfred/data/wrench.md
@@ -64,4 +64,6 @@ returns the string representation of the dataset
 ```python
 def __repr__(self):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/ai21.md b/docs/alfred/fm/ai21.md
@@ -24,4 +24,6 @@ This class provides a wrapper for the OpenAI API for generating completions.
 class AI21Model(APIAccessFoundationModel):
     def __init__(self, model_string: str = "j1-large", api_key: Optional[str] = None):
         ...
-```
+```
+
+
diff --git a/docs/alfred/fm/anthropic.md b/docs/alfred/fm/anthropic.md
@@ -38,4 +38,6 @@ Launch an interactive chat session with the Anthropic API.
 ```python
 def chat(self, **kwargs: Any):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/cohere.md b/docs/alfred/fm/cohere.md
@@ -24,4 +24,6 @@ This class provides a wrapper for the OpenAI API for generating completions.
 class CohereModel(APIAccessFoundationModel):
     def __init__(self, model_string: str = "xlarge", api_key: Optional[str] = None):
         ...
-```
+```
+
+
diff --git a/docs/alfred/fm/dummy.md b/docs/alfred/fm/dummy.md
@@ -29,4 +29,6 @@ class DummyModel(LocalAccessFoundationModel):
 
 #### See also
 
-- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+
+
diff --git a/docs/alfred/fm/flexgen.md b/docs/alfred/fm/flexgen.md
@@ -0,0 +1,44 @@
+# Flexgen
+
+[Alfred Index](../../README.md#alfred-index) /
+[Alfred](../index.md#alfred) /
+[Fm](./index.md#fm) /
+Flexgen
+
+> Auto-generated documentation for [alfred.fm.flexgen](../../../alfred/fm/flexgen.py) module.
+
+- [Flexgen](#flexgen)
+  - [FlexGenModel](#flexgenmodel)
+
+## FlexGenModel
+
+[Show source in flexgen.py:13](../../../alfred/fm/flexgen.py#L13)
+
+FlexGenModel wraps a FlexGen model. FlexGen is used for High-throughput generative inference with single GPU.
+
+Currently, FlexGen supports OPT style models.
+
+source: https://github.com/FMInference/FlexGen
+paper: https://arxiv.org/pdf/2303.06865.pdf
+
+#### Signature
+
+```python
+class FlexGenModel(LocalAccessFoundationModel):
+    def __init__(
+        self,
+        model: str,
+        local_dir: str,
+        model_string: str,
+        policy: Union[List, Policy] = (100, 0, 100, 0, 100, 0),
+        offload_dir: str = "./flexgen_offload_cache",
+        **kwargs: Any
+    ):
+        ...
+```
+
+#### See also
+
+- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+
+
diff --git a/docs/alfred/fm/huggingface.md b/docs/alfred/fm/huggingface.md
@@ -45,4 +45,6 @@ class HuggingFaceModel(LocalAccessFoundationModel):
 
 #### See also
 
-- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+
+
diff --git a/docs/alfred/fm/huggingfacevlm.md b/docs/alfred/fm/huggingfacevlm.md
@@ -33,4 +33,6 @@ class HuggingFaceCLIPModel(LocalAccessFoundationModel):
 
 #### See also
 
-- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+
+
diff --git a/docs/alfred/fm/model.md b/docs/alfred/fm/model.md
@@ -286,4 +286,6 @@ class LocalAccessFoundationModel(FoundationModel):
 
 #### See also
 
-- [FoundationModel](#foundationmodel)
+- [FoundationModel](#foundationmodel)
+
+
diff --git a/docs/alfred/fm/onnx.md b/docs/alfred/fm/onnx.md
@@ -30,4 +30,6 @@ class ONNXModel(LocalAccessFoundationModel):
 
 #### See also
 
-- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+- [LocalAccessFoundationModel](./model.md#localaccessfoundationmodel)
+
+
diff --git a/docs/alfred/fm/openai.md b/docs/alfred/fm/openai.md
@@ -13,7 +13,7 @@ Openai
 
 ## OpenAIModel
 
-[Show source in openai.py:49](../../../alfred/fm/openai.py#L49)
+[Show source in openai.py:50](../../../alfred/fm/openai.py#L50)
 
 A wrapper for the OpenAI API.
 
@@ -31,7 +31,7 @@ class OpenAIModel(APIAccessFoundationModel):
 
 ### OpenAIModel().chat
 
-[Show source in openai.py:228](../../../alfred/fm/openai.py#L228)
+[Show source in openai.py:229](../../../alfred/fm/openai.py#L229)
 
 Launch an interactive chat session with the OpenAI API.
 
@@ -40,4 +40,6 @@ Launch an interactive chat session with the OpenAI API.
 ```python
 def chat(self, **kwargs: Any):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/query/completion_query.md b/docs/alfred/fm/query/completion_query.md
@@ -143,4 +143,6 @@ returns the raw prompt content
 @property
 def prompt(self):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/query/query.md b/docs/alfred/fm/query/query.md
@@ -82,4 +82,6 @@ Type: *str*
 ```python
 def serialize(self) -> str:
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/query/ranked_query.md b/docs/alfred/fm/query/ranked_query.md
@@ -195,4 +195,6 @@ returns the raw prompt content
 @property
 def prompt(self):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/remote/grpc.md b/docs/alfred/fm/remote/grpc.md
@@ -132,4 +132,6 @@ def restart(self):
 ```python
 def serve(self, credentials: Optional[grpc.ServerCredentials] = None):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/remote/protos/query_pb2.md b/docs/alfred/fm/remote/protos/query_pb2.md
@@ -8,4 +8,5 @@
 Query Pb2
 
 > Auto-generated documentation for [alfred.fm.remote.protos.query_pb2](../../../../../alfred/fm/remote/protos/query_pb2.py) module.
+
 - [Query Pb2](#query-pb2)
diff --git a/docs/alfred/fm/remote/protos/query_pb2_grpc.md b/docs/alfred/fm/remote/protos/query_pb2_grpc.md
@@ -146,4 +146,6 @@ class QueryServiceStub(object):
 ```python
 def add_QueryServiceServicer_to_server(servicer, server):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/remote/utils.md b/docs/alfred/fm/remote/utils.md
@@ -76,4 +76,6 @@ def port_finder(port: int) -> int:
 ```python
 def tensor_to_bytes(tensor):
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/response/completion_response.md b/docs/alfred/fm/response/completion_response.md
@@ -120,4 +120,6 @@ Type: *float*
 @property
 def score(self) -> Dict:
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/response/ranked_response.md b/docs/alfred/fm/response/ranked_response.md
@@ -137,4 +137,6 @@ Type: *dict*
 @property
 def scores(self) -> Dict:
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/response/response.md b/docs/alfred/fm/response/response.md
@@ -100,4 +100,6 @@ Type: *str*
 ```python
 def serialize(self) -> str:
     ...
-```
+```
+
+
diff --git a/docs/alfred/fm/utils.md b/docs/alfred/fm/utils.md
@@ -47,7 +47,7 @@ class DynamicBatcher:
 
 ### DynamicBatcher().batch
 
-[Show source in utils.py:375](../../../alfred/fm/utils.py#L375)
+[Show source in utils.py:377](../../../alfred/fm/utils.py#L377)
 
 Batch a list of instances into a list of batches.
 If the instances are of different sizes, they will be sorted by size
@@ -67,7 +67,7 @@ def batch(self) -> List:
 
 ### DynamicBatcher().merge_rank_response
 
-[Show source in utils.py:289](../../../alfred/fm/utils.py#L289)
+[Show source in utils.py:291](../../../alfred/fm/utils.py#L291)
 
 Merge a list of responses with raw logit into a single RankedResponse
 Assumption: Candidate Order is the same across all ranked queries
@@ -95,7 +95,7 @@ def merge_rank_response(
 
 ### DynamicBatcher().reorder
 
-[Show source in utils.py:334](../../../alfred/fm/utils.py#L334)
+[Show source in utils.py:336](../../../alfred/fm/utils.py#L336)
 
 Reordering the responses according to the original order of the queries
 
@@ -361,4 +361,6 @@ Type: *List[int]*
 ```python
 def tokenize(inst, tokenizer, max_length=512):
     ...
-```
+```
+
+
diff --git a/docs/alfred/labeling/flyingsquid.md b/docs/alfred/labeling/flyingsquid.md
@@ -34,4 +34,6 @@ class FlyingSquid(LabelModel):
 ```python
 def label(self, votes: np.ndarray) -> np.ndarray:
     ...
-```
+```
+
+
diff --git a/docs/alfred/labeling/labelmodel.md b/docs/alfred/labeling/labelmodel.md
@@ -49,4 +49,6 @@ def __call__(self, votes):
 @abc.abstractmethod
 def label(self, votes):
     ...
-```
+```
+
+
diff --git a/docs/alfred/labeling/majority_vote.md b/docs/alfred/labeling/majority_vote.md
@@ -36,4 +36,6 @@ returns the majority vote for each response row
 ```python
 def label(self, votes: np.ndarray) -> np.ndarray:
     ...
-```
+```
+
+
diff --git a/docs/alfred/labeling/naive_bayes.md b/docs/alfred/labeling/naive_bayes.md
@@ -47,4 +47,6 @@ Type: *np.ndarray*
 ```python
 def label(self, votes: np.ndarray) -> np.ndarray:
     ...
-```
+```
+
+