Add --list-attacks argument that lists the available attacks and thei…

…r descriptions

README.md

@@ -71,7 +71,7 @@ We use a dynamic testing approach, where we get the necessary context from your
 ### Usage
 
 ```
-usage: psfuzz.py [-h] [-l] [--attack-provider ATTACK_PROVIDER] [--attack-model ATTACK_MODEL]
+usage: psfuzz.py [-h] [--list-providers] [--attack-provider ATTACK_PROVIDER] [--attack-model ATTACK_MODEL]
                  [--target-provider TARGET_PROVIDER] [--target-model TARGET_MODEL] [-n NUM_ATTACKS]
                  [-d DEBUG_LEVEL] [-i]
                  [system_prompt_file]
@@ -83,7 +83,7 @@ positional arguments:
 
 options:
   -h, --help            show this help message and exit
-  -l, --list-providers  List available providers and exit
+  --list-providers  List available providers and exit
   --attack-provider ATTACK_PROVIDER
                         Attack provider (default: 'open_ai')
   --attack-model ATTACK_MODEL

ps_fuzz/chat_clients.py

@@ -23,6 +23,10 @@ def interact(self, history: MessageList, messages: MessageList) -> BaseMessage:
            The history is automatically updated during conversation.
         """
 
+class FakeChatClient(ClientBase):
+    def interact(self, history: MessageList, messages: MessageList) -> BaseMessage:
+        return "FakeChat response"
+
 # Specialized chat client based on langchain supported backends
 class ClientLangChain(ClientBase):
     "Chat model wrapper around LangChain"

ps_fuzz/cli.py

@@ -36,7 +36,8 @@ def setup_logging(logging_level):
 def main():
     # Parse command line arguments
     parser = argparse.ArgumentParser(description='Prompt Security LLM Prompt Injection Fuzzer')
-    parser.add_argument('-l', '--list-providers', action='store_true', default=False, help="List available providers and exit")
+    parser.add_argument('--list-providers', action='store_true', default=False, help="List available providers and exit")
+    parser.add_argument('--list-attacks', action='store_true', default=False, help="List available attacks and exit")
     parser.add_argument('--attack-provider', type=str, default="open_ai", help="Attack provider (default: 'open_ai')")
     parser.add_argument('--attack-model', type=str, default="gpt-3.5-turbo", help="Attack model (default: 'gpt-3.5-turbo')")
     parser.add_argument('--target-provider', type=str, default="open_ai", help="Target provider (default: 'open_ai')")
@@ -56,6 +57,15 @@ def main():
             print(f"  {BRIGHT}{provider_name}{RESET}: {provider_info.short_doc}")
         sys.exit(0)
 
+    if args.list_attacks:
+        client_config = ClientConfig(FakeChatClient(), [])
+        attack_config = AttackConfig(FakeChatClient(), 1)
+        tests = instantiate_tests(client_config, attack_config)
+        print("Available attacks:")
+        for test_name, test_description in [(cls.test_name, cls.test_description) for cls in tests]:
+            print(f"  {BRIGHT}{test_name}{RESET}: {test_description}")
+        sys.exit(0)
+
     if args.system_prompt_file is None:
         print(f"Error: You must specify the name of a text file containing your system prompt.\n", file=sys.stderr)
         parser.print_help()