InternScience · ChenZiHong-Gavin · Dec 3, 2025 · Dec 3, 2025 · Dec 3, 2025 · Dec 3, 2025
diff --git a/scripts/baselines/generate_all_baselines.sh → examples/baselines/generate_all_baselines.sh b/scripts/baselines/generate_all_baselines.sh → examples/baselines/generate_all_baselines.sh
diff --git a/scripts/baselines/generate_bds.sh → examples/baselines/generate_bds.sh b/scripts/baselines/generate_bds.sh → examples/baselines/generate_bds.sh
diff --git a/scripts/baselines/generate_entigraph.sh → examples/baselines/generate_entigraph.sh b/scripts/baselines/generate_entigraph.sh → examples/baselines/generate_entigraph.sh
diff --git a/scripts/baselines/generate_genie.sh → examples/baselines/generate_genie.sh b/scripts/baselines/generate_genie.sh → examples/baselines/generate_genie.sh
diff --git a/scripts/baselines/generate_longform.sh → examples/baselines/generate_longform.sh b/scripts/baselines/generate_longform.sh → examples/baselines/generate_longform.sh
diff --git a/scripts/baselines/generate_selfqa.sh → examples/baselines/generate_selfqa.sh b/scripts/baselines/generate_selfqa.sh → examples/baselines/generate_selfqa.sh
diff --git a/scripts/baselines/generate_wrap.sh → examples/baselines/generate_wrap.sh b/scripts/baselines/generate_wrap.sh → examples/baselines/generate_wrap.sh
diff --git a/scripts/evaluate/evaluate.sh → examples/evaluate/evaluate.sh b/scripts/evaluate/evaluate.sh → examples/evaluate/evaluate.sh
diff --git a/examples/extract/extract_schema_guided/README.md b/examples/extract/extract_schema_guided/README.md
@@ -0,0 +1 @@
+# Extract Schema-Guided Information from Documents
diff --git a/examples/extract/extract_schema_guided/extract_schema_guided.sh b/examples/extract/extract_schema_guided/extract_schema_guided.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/extract/extract_schema_guided/schema_guided_extraction_config.yaml \
+--output_dir cache/
diff --git a/examples/extract/extract_schema_guided/schema_guided_extraction_config.yaml b/examples/extract/extract_schema_guided/schema_guided_extraction_config.yaml
@@ -0,0 +1,34 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/extract_demo.txt
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 20480 # larger chunk size for better context
+      chunk_overlap: 2000
+
+  - id: extract
+    op_name: extract
+    type: map_batch
+    dependencies:
+      - chunk
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: schema_guided
+      schema_path: graphgen/templates/extraction/schemas/legal_contract.json
diff --git a/examples/generate/generate_aggregated_qa/README.md b/examples/generate/generate_aggregated_qa/README.md
@@ -0,0 +1,3 @@
+# Generate Aggregated QAs
+
+Aggregated mode is one of three question-answering scenarios in GraphGen (alongside atomic and multi-hop) designed to generate synthetic training data that incorporates complex, integrated knowledge from multiple sources.
diff --git a/examples/generate/generate_aggregated_qa/aggregated_config.yaml b/examples/generate/generate_aggregated_qa/aggregated_config.yaml
@@ -0,0 +1,77 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read_files # id is unique in the pipeline, and can be referenced by other steps
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+        input_path:
+          - examples/input_examples/jsonl_demo.jsonl # input file path, support json, jsonl, txt, pdf. See examples/input_examples for examples
+
+  - id: chunk_documents
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read_files
+    execution_params:
+      replicas: 4
+    params:
+        chunk_size: 1024 # chunk size for text splitting
+        chunk_overlap: 100 # chunk overlap for text splitting
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    dependencies:
+      - chunk_documents
+    execution_params:
+      replicas: 1
+      batch_size: 128
+
+  - id: quiz
+    op_name: quiz
+    type: aggregate
+    dependencies:
+      - build_kg
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      quiz_samples: 2 # number of quiz samples to generate
+      concurrency_limit: 200
+
+  - id: judge
+    op_name: judge
+    type: map_batch
+    dependencies:
+      - quiz
+    execution_params:
+      replicas: 1
+      batch_size: 128
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - judge
+    params:
+      method: ece # ece is a custom partition method based on comprehension loss
+      method_params:
+        max_units_per_community: 20 # max nodes and edges per community
+        min_units_per_community: 5 # min nodes and edges per community
+        max_tokens_per_community: 10240 # max tokens per community
+        unit_sampling: max_loss # unit sampling strategy, support: random, max_loss, min_loss
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: aggregated # atomic, aggregated, multi_hop, cot, vqa
+      data_format: ChatML # Alpaca, Sharegpt, ChatML
diff --git a/examples/generate/generate_aggregated_qa/generate_aggregated.sh b/examples/generate/generate_aggregated_qa/generate_aggregated.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/generate/generate_aggregated_qa/aggregated_config.yaml \
+--output_dir cache/
diff --git a/examples/generate/generate_atomic_qa/README.md b/examples/generate/generate_atomic_qa/README.md
@@ -0,0 +1,3 @@
+# Generate Atomic QAs
+
+Atomic mode generates question-answer pairs that test basic, isolated knowledge from individual facts or relationships in the knowledge graph. 
diff --git a/examples/generate/generate_atomic_qa/atomic_config.yaml b/examples/generate/generate_atomic_qa/atomic_config.yaml
@@ -0,0 +1,53 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/json_demo.json
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 1024
+      chunk_overlap: 100
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    dependencies:
+      - chunk
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - build_kg
+    params:
+      method: dfs
+      method_params:
+        max_units_per_community: 1
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: atomic
+      data_format: Alpaca
diff --git a/examples/generate/generate_atomic_qa/generate_atomic.sh b/examples/generate/generate_atomic_qa/generate_atomic.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/generate/generate_atomic_qa/atomic_config.yaml \
+--output_dir cache/
diff --git a/examples/generate/generate_cot_qa/README.md b/examples/generate/generate_cot_qa/README.md
@@ -0,0 +1 @@
+# Generate CoT QAs
diff --git a/examples/generate/generate_cot_qa/cot_config.yaml b/examples/generate/generate_cot_qa/cot_config.yaml
@@ -0,0 +1,55 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/txt_demo.txt
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 1024
+      chunk_overlap: 100
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    dependencies:
+      - chunk
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - build_kg
+    params:
+      method: leiden
+      method_params:
+        max_size: 20
+        use_lcc: false
+        random_seed: 42
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: cot
+      data_format: Sharegpt
diff --git a/examples/generate/generate_cot_qa/generate_cot.sh b/examples/generate/generate_cot_qa/generate_cot.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/generate/generate_cot_qa/cot_config.yaml \
+--output_dir cache/
diff --git a/examples/generate/generate_multi_hop_qa/README.md b/examples/generate/generate_multi_hop_qa/README.md
@@ -0,0 +1 @@
+# Generate Multi-hop QAs
diff --git a/examples/generate/generate_multi_hop_qa/generate_multi_hop.sh b/examples/generate/generate_multi_hop_qa/generate_multi_hop.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/generate/generate_multi_hop_qa/multi_hop_config.yaml \
+--output_dir cache/
diff --git a/examples/generate/generate_multi_hop_qa/multi_hop_config.yaml b/examples/generate/generate_multi_hop_qa/multi_hop_config.yaml
@@ -0,0 +1,56 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/csv_demo.csv
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 1024
+      chunk_overlap: 100
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    dependencies:
+      - chunk
+    execution_params:
+      replicas: 1
+      batch_size: 128
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - build_kg
+    params:
+      method: ece
+      method_params:
+        max_units_per_community: 3
+        min_units_per_community: 3
+        max_tokens_per_community: 10240
+        unit_sampling: random
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: multi_hop
+      data_format: ChatML
diff --git a/examples/generate/generate_vqa/README.md b/examples/generate/generate_vqa/README.md
@@ -0,0 +1 @@
+# Generate VQAs
diff --git a/examples/generate/generate_vqa/generate_vqa.sh b/examples/generate/generate_vqa/generate_vqa.sh
@@ -0,0 +1,3 @@
+python3 -m graphgen.run \
+--config_file examples/generate/generate_vqa/vqa_config.yaml \
+--output_dir cache/
diff --git a/examples/generate/generate_vqa/vqa_config.yaml b/examples/generate/generate_vqa/vqa_config.yaml
@@ -0,0 +1,57 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/vqa_demo.json
+      modalities:
+        - text
+        - image
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 1024
+      chunk_overlap: 100
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    dependencies:
+      - chunk
+    execution_params:
+      replicas: 1
+      batch_size: 128
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - build_kg
+    params:
+      method: anchor_bfs
+      method_params:
+        anchor_type: image
+        max_units_per_community: 10
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: vqa
+      data_format: ChatML
diff --git a/resources/input_examples/csv_demo.csv → examples/input_examples/csv_demo.csv b/resources/input_examples/csv_demo.csv → examples/input_examples/csv_demo.csv
diff --git a/resources/input_examples/extract_demo.txt → examples/input_examples/extract_demo.txt b/resources/input_examples/extract_demo.txt → examples/input_examples/extract_demo.txt
diff --git a/...urces/input_examples/graphml_demo.graphml → examples/input_examples/graphml_demo.graphml b/...urces/input_examples/graphml_demo.graphml → examples/input_examples/graphml_demo.graphml
diff --git a/...3064cf17c5435814edfbee42ae6b19aac37d2.jpg → ...3064cf17c5435814edfbee42ae6b19aac37d2.jpg b/...3064cf17c5435814edfbee42ae6b19aac37d2.jpg → ...3064cf17c5435814edfbee42ae6b19aac37d2.jpg
diff --git a/...ee99e96ffa8b6df4476c9b12d7bb1dd20d635.jpg → ...ee99e96ffa8b6df4476c9b12d7bb1dd20d635.jpg b/...ee99e96ffa8b6df4476c9b12d7bb1dd20d635.jpg → ...ee99e96ffa8b6df4476c9b12d7bb1dd20d635.jpg
diff --git a/...e2ae309fee014082db00bc2d87187a6bb5dca.jpg → ...e2ae309fee014082db00bc2d87187a6bb5dca.jpg b/...e2ae309fee014082db00bc2d87187a6bb5dca.jpg → ...e2ae309fee014082db00bc2d87187a6bb5dca.jpg
diff --git a/...7df02964c9c3da8d8e9567ea19240b14cc742.jpg → ...7df02964c9c3da8d8e9567ea19240b14cc742.jpg b/...7df02964c9c3da8d8e9567ea19240b14cc742.jpg → ...7df02964c9c3da8d8e9567ea19240b14cc742.jpg
diff --git a/...ffe56f793f287b3399345aea31cd20eed2824.jpg → ...ffe56f793f287b3399345aea31cd20eed2824.jpg b/...ffe56f793f287b3399345aea31cd20eed2824.jpg → ...ffe56f793f287b3399345aea31cd20eed2824.jpg
diff --git a/...a0129a0475b2ab5b920a4cff20a4fb623517d.jpg → ...a0129a0475b2ab5b920a4cff20a4fb623517d.jpg b/...a0129a0475b2ab5b920a4cff20a4fb623517d.jpg → ...a0129a0475b2ab5b920a4cff20a4fb623517d.jpg
diff --git a/resources/input_examples/json_demo.json → examples/input_examples/json_demo.json b/resources/input_examples/json_demo.json → examples/input_examples/json_demo.json
diff --git a/resources/input_examples/jsonl_demo.jsonl → examples/input_examples/jsonl_demo.jsonl b/resources/input_examples/jsonl_demo.jsonl → examples/input_examples/jsonl_demo.jsonl
diff --git a/resources/input_examples/pdf_demo.pdf → examples/input_examples/pdf_demo.pdf b/resources/input_examples/pdf_demo.pdf → examples/input_examples/pdf_demo.pdf
diff --git a/...rces/input_examples/search_dna_demo.jsonl → ...ples/input_examples/search_dna_demo.jsonl b/...rces/input_examples/search_dna_demo.jsonl → ...ples/input_examples/search_dna_demo.jsonl
diff --git a/.../input_examples/search_protein_demo.jsonl → .../input_examples/search_protein_demo.jsonl b/.../input_examples/search_protein_demo.jsonl → .../input_examples/search_protein_demo.jsonl
diff --git a/...rces/input_examples/search_rna_demo.jsonl → ...ples/input_examples/search_rna_demo.jsonl b/...rces/input_examples/search_rna_demo.jsonl → ...ples/input_examples/search_rna_demo.jsonl
diff --git a/resources/input_examples/txt_demo.txt → examples/input_examples/txt_demo.txt b/resources/input_examples/txt_demo.txt → examples/input_examples/txt_demo.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		# Extract Schema-Guided Information from Documents
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Generate Aggregated QAs

		Aggregated mode is one of three question-answering scenarios in GraphGen (alongside atomic and multi-hop) designed to generate synthetic training data that incorporates complex, integrated knowledge from multiple sources.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Generate Atomic QAs

		Atomic mode generates question-answer pairs that test basic, isolated knowledge from individual facts or relationships in the knowledge graph.