refactor: secure subprocess calls and fix linting failures

rdagent/components/coder/factor_coder/factor.py

@@ -161,8 +161,7 @@ def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
 
             try:
                 subprocess.check_output(
-                    f"{FACTOR_COSTEER_SETTINGS.python_bin} {execution_code_path}",
-                    shell=True,
+                    [str(FACTOR_COSTEER_SETTINGS.python_bin), str(execution_code_path)],
                     cwd=self.workspace_path,
                     stderr=subprocess.STDOUT,
                     timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout,

rdagent/log/ui/app.py

@@ -561,11 +561,9 @@ def research_window():
             if hg := state.msgs[round]["hypothesis generation"]:
                 st.markdown("**Hypothesis💡**")  # 🧠
                 h: Hypothesis = hg[0].content
-                st.markdown(
-                    f"""
+                st.markdown(f"""
 - **Hypothesis**: {h.hypothesis}
-- **Reason**: {h.reason}"""
-                )
+- **Reason**: {h.reason}""")
 
             if eg := state.msgs[round]["experiment generation"]:
                 tasks_window(eg[0].content)
@@ -644,14 +642,12 @@ def feedback_window():
                     st.plotly_chart(fig)
                 st.markdown("**Hypothesis Feedback🔍**")
                 h: HypothesisFeedback = fb[0].content
-                st.markdown(
-                    f"""
+                st.markdown(f"""
 - **Observations**: {h.observations}
 - **Hypothesis Evaluation**: {h.hypothesis_evaluation}
 - **New Hypothesis**: {h.new_hypothesis}
 - **Decision**: {h.decision}
-- **Reason**: {h.reason}"""
-                )
+- **Reason**: {h.reason}""")
 
             if isinstance(state.scenario, KGScenario):
                 if fbe := state.msgs[round]["runner result"]:

rdagent/log/ui/ds_trace.py

@@ -189,13 +189,11 @@ def task_win(task):
             st.markdown(f"**:blue[Package Info:]**")
             st.code(task.package_info)
         if hasattr(task, "architecture"):  # model task
-            st.markdown(
-                f"""
+            st.markdown(f"""
     | Model_type | Architecture | hyperparameters |
     |------------|--------------|-----------------|
     | {task.model_type} | {task.architecture} | {task.hyperparameters} |
-    """
-            )
+    """)
 
 
 def workspace_win(workspace, cmp_workspace=None, cmp_name="last code."):
@@ -1168,17 +1166,15 @@ def get_folders_sorted(log_path, sort_by_time=False):
     st.toggle("**Show LLM Log**", key="show_llm_log")
     st.toggle("*Show stdout*", key="show_stdout")
     st.toggle("*Show save workspace*", key="show_save_input")
-    st.markdown(
-        f"""
+    st.markdown(f"""
 - [Summary](#summary)
 - [Exp Gen](#exp-gen)
 - [Coding](#coding)
 - [Running](#running)
 - [Feedback](#feedback)
 - [Record](#record)
     - [SOTA Experiment](#sota-exp)
-"""
-    )
+""")
 
 
 def get_state_data_range(state_data):

rdagent/log/ui/web.py

@@ -171,26 +171,22 @@ def consume_msg(self, msg: Message | Hypothesis):
         h: Hypothesis = msg.content if isinstance(msg, Message) else msg
 
         self.container.markdown("#### **Hypothesis💡**")
-        self.container.markdown(
-            f"""
+        self.container.markdown(f"""
 - **Hypothesis**: {h.hypothesis}
-- **Reason**: {h.reason}"""
-        )
+- **Reason**: {h.reason}""")
 
 
 class HypothesisFeedbackWindow(StWindow):
     def consume_msg(self, msg: Message | HypothesisFeedback):
         h: HypothesisFeedback = msg.content if isinstance(msg, Message) else msg
 
         self.container.markdown("#### **Hypothesis Feedback🔍**")
-        self.container.markdown(
-            f"""
+        self.container.markdown(f"""
 - **Observations**: {h.observations}
 - **Hypothesis Evaluation**: {h.hypothesis_evaluation}
 - **New Hypothesis**: {h.new_hypothesis}
 - **Decision**: {h.decision}
-- **Reason**: {h.reason}"""
-        )
+- **Reason**: {h.reason}""")
 
 
 class FactorTaskWindow(StWindow):
@@ -225,8 +221,7 @@ class FactorFeedbackWindow(StWindow):
     def consume_msg(self, msg: Message | FactorSingleFeedback):
         fb: FactorSingleFeedback = msg.content if isinstance(msg, Message) else msg
 
-        self.container.markdown(
-            f"""### :blue[Factor Execution Feedback]
+        self.container.markdown(f"""### :blue[Factor Execution Feedback]
 {fb.execution_feedback}
 ### :blue[Factor Code Feedback]
 {fb.code_feedback}
@@ -236,16 +231,14 @@ def consume_msg(self, msg: Message | FactorSingleFeedback):
 {fb.final_feedback}
 ### :blue[Factor Final Decision]
 This implementation is {'SUCCESS' if fb.final_decision else 'FAIL'}.
-"""
-        )
+""")
 
 
 class ModelFeedbackWindow(StWindow):
     def consume_msg(self, msg: Message | ModelSingleFeedback):
         mb: ModelSingleFeedback = msg.content if isinstance(msg, Message) else msg
 
-        self.container.markdown(
-            f"""### :blue[Model Execution Feedback]
+        self.container.markdown(f"""### :blue[Model Execution Feedback]
 {mb.execution_feedback}
 ### :blue[Model Shape Feedback]
 {mb.shape_feedback}
@@ -257,8 +250,7 @@ def consume_msg(self, msg: Message | ModelSingleFeedback):
 {mb.final_feedback}
 ### :blue[Model Final Decision]
 This implementation is {'SUCCESS' if mb.final_decision else 'FAIL'}.
-"""
-        )
+""")
 
 
 class WorkspaceWindow(StWindow):

rdagent/oai/backend/base.py

@@ -483,7 +483,9 @@ def _try_create_chat_completion_or_embedding(  # type: ignore[no-untyped-def]
                     kwargs["add_json_in_prompt"] = True
 
                 too_long_error_message = hasattr(e, "message") and (
-                    "maximum context length" in e.message or "input must have less than" in e.message
+                    "maximum context length" in e.message
+                    or "input must have less than" in e.message
+                    or "Request contains an invalid argument" in e.message
                 )
 
                 if embedding and too_long_error_message:

rdagent/scenarios/data_science/proposal/exp_gen/proposal.py

@@ -649,10 +649,8 @@ def hypothesis_gen(
 
         # knowledge retrieval
         if DS_RD_SETTING.enable_research_rag:
-            rag_agent = RAGAgent(
-                system_prompt="""You are a helpful assistant.
-You help users retrieve relevant knowledge from community discussions and public code."""
-            )
+            rag_agent = RAGAgent(system_prompt="""You are a helpful assistant.
+You help users retrieve relevant knowledge from community discussions and public code.""")
             knowledge = rag_agent.query(problem_formatted_str)
         else:
             knowledge = None

rdagent/scenarios/qlib/experiment/utils.py

@@ -67,17 +67,15 @@ def get_file_desc(p: Path, variable_list=[]) -> str:
     """
     p = Path(p)
 
-    JJ_TPL = Environment(undefined=StrictUndefined).from_string(
-        """
+    JJ_TPL = Environment(undefined=StrictUndefined).from_string("""
 # {{file_name}}
 
 ## File Type
 {{type_desc}}
 
 ## Content Overview
 {{content}}
-"""
-    )
+""")
 
     if p.name.endswith(".h5"):
         df = pd.read_hdf(p)

rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py

@@ -463,9 +463,7 @@ def __deduplicate_factor_dict(factor_dict: dict[str, dict[str, str]]) -> list[li
         description = factor_dict[factor_name]["description"]
         formulation = factor_dict[factor_name]["formulation"]
         variables = factor_dict[factor_name]["variables"]
-        factor_name_to_full_str[
-            factor_name
-        ] = f"""Factor name: {factor_name}
+        factor_name_to_full_str[factor_name] = f"""Factor name: {factor_name}
 Factor description: {description}
 Factor formulation: {formulation}
 Factor variables: {variables}

test/oai/test_embedding_and_similarity.py

@@ -30,13 +30,10 @@ def test_embedding_long_text_truncation(self) -> None:
         """Test embedding with very long text that exceeds token limits"""
         # Create a very long text that will definitely exceed embedding token limits
         # Using a repetitive pattern to simulate a real long document
-        long_content = (
-            """
+        long_content = """
         This is a very long document that contains a lot of repetitive content to test the embedding truncation functionality.
         We need to make this text long enough to exceed the typical embedding model token limits of around 8192 tokens.
-        """
-            * 1000
-        )  # This should create a text with approximately 50,000+ tokens
+        """ * 1000  # This should create a text with approximately 50,000+ tokens
         # This should trigger the gradual truncation mechanism
         emb = APIBackend().create_embedding(long_content)