feat(io): add import/export functionality for JSON and CSV

atasoglu · atasoglu · commit cb413f882d36 · 2025-10-29T19:48:35.000+03:00
- Implemented methods for exporting and importing data in JSON Lines and CSV formats.
- Added support for filtered exports using metadata filters.
- Included batch processing for efficient handling of large datasets.
- Created comprehensive tests for import/export functionality.
- Updated documentation and examples to reflect new features.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,14 +18,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Comprehensive test coverage for metadata filtering (unit, integration, and security tests)
 - New example `advanced_metadata_queries.py` demonstrating nested paths and complex queries
 - Updated `metadata_filtering.py` example with new filtering methods
+- **Export/import functionality**: New methods for data portability
+  - `export_to_json()` - Export records to JSON Lines format
+  - `import_from_json()` - Import records from JSON Lines format
+  - `export_to_csv()` - Export records to CSV format
+  - `import_from_csv()` - Import records from CSV format
+- Support for filtered exports using metadata filters
+- Batch processing for memory-efficient import/export of large datasets
+- Optional embedding inclusion in exports
+- New `io` module with import/export utilities
+- Comprehensive tests for import/export functionality
+- New example `export_import_example.py` demonstrating backup/restore workflows
 
 ### Security
 - SQL injection prevention in metadata filter keys
 - Validation of JSON paths to prevent malicious queries
 - Parameterized queries for all metadata filtering operations
 
+### Improved
+- Data migration workflows now much easier
+- Backup and restore capabilities for production use
+- Interoperability with external systems via CSV/JSON
+
 ### Documentation
 - Added "Metadata Filtering" section to README with examples
+- Added "Export/Import" section to README with examples
 - Updated examples list in README
 - Added comprehensive docstrings for new methods
 
@@ -190,7 +207,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Version History
 
-- **2.2.0** - Added metadata filtering with JSON_EXTRACT support
+- **2.2.0** - Added metadata filtering with JSON_EXTRACT support and export/import functionality
 - **2.1.1** - Moved table name validation to create_table()
 - **2.1.0** - Added connection pooling support
 - **2.0.0** - Major refactor: simplified API, removed niche methods, cleaner naming
diff --git a/README.md b/README.md
@@ -66,6 +66,37 @@ rows = client.get_many(rowids)
 client.close()
 ```
 
+## Export/Import
+
+Export and import data in JSON or CSV formats for backups, migrations, and data sharing:
+
+```python
+# Export to JSON (includes embeddings)
+count = client.export_to_json("backup.jsonl")
+
+# Export to CSV (human-readable, optional embeddings)
+count = client.export_to_csv("data.csv", include_embeddings=False)
+
+# Export filtered data
+count = client.export_to_json(
+    "important.jsonl",
+    filters={"priority": "high"}
+)
+
+# Import from JSON
+count = client.import_from_json("backup.jsonl")
+
+# Import from CSV
+count = client.import_from_csv("data.csv")
+
+# Backup and restore workflow
+client.export_to_json("backup.jsonl")
+# ... data loss ...
+client.import_from_json("backup.jsonl")
+```
+
+See [examples/export_import_example.py](examples/export_import_example.py) for more examples.
+
 ## Metadata Filtering
 
 Efficiently filter records by metadata fields using SQLite's JSON functions:
@@ -255,6 +286,7 @@ Edit [benchmarks/config.yaml](benchmarks/config.yaml) to customize:
   - [basic_usage.py](examples/basic_usage.py) - Basic CRUD operations
   - [metadata_filtering.py](examples/metadata_filtering.py) - Metadata filtering and queries
   - [advanced_metadata_queries.py](examples/advanced_metadata_queries.py) - Advanced metadata filtering with nested paths
+  - [export_import_example.py](examples/export_import_example.py) - Export/import data in JSON and CSV formats
   - [transaction_example.py](examples/transaction_example.py) - Transaction management with all CRUD operations
   - [batch_operations.py](examples/batch_operations.py) - Bulk operations
   - [logging_example.py](examples/logging_example.py) - Logging configuration
diff --git a/TODO b/TODO
@@ -72,8 +72,8 @@
 - [x] Partial search on JSON metadata (JSON_EXTRACT)
 - [x] Metadata field filtering (key-value based)
 - [x] Transaction context manager
+- [x] Export/import functions (JSON, CSV)
 - [ ] Async/await support (aiosqlite)
-- [ ] Export/import functions (JSON, CSV)
 - [ ] Table migration utilities
 - [ ] Backup/restore functions
 
diff --git a/examples/export_import_example.py b/examples/export_import_example.py
@@ -0,0 +1,123 @@
+"""Export/import example for sqlite-vec-client.
+
+Demonstrates:
+- Exporting data to JSON and CSV
+- Importing data from JSON and CSV
+- Filtered exports
+- Backup and restore workflows
+"""
+
+from sqlite_vec_client import SQLiteVecClient
+
+
+def main():
+    # Create and populate database
+    client = SQLiteVecClient(table="documents", db_path=":memory:")
+    client.create_table(dim=128, distance="cosine")
+
+    texts = [
+        "Introduction to Python",
+        "Advanced JavaScript",
+        "Python for Data Science",
+        "Java Programming Guide",
+        "Machine Learning with Python",
+    ]
+
+    embeddings = [[0.1 * i] * 128 for i in range(len(texts))]
+
+    metadata = [
+        {"category": "python", "level": "beginner"},
+        {"category": "javascript", "level": "advanced"},
+        {"category": "python", "level": "intermediate"},
+        {"category": "java", "level": "beginner"},
+        {"category": "python", "level": "advanced"},
+    ]
+
+    client.add(texts=texts, embeddings=embeddings, metadata=metadata)
+    print(f"Added {client.count()} documents\n")
+
+    # Example 1: Export all data to JSON
+    print("=== Export to JSON ===")
+    count = client.export_to_json("backup.jsonl")
+    print(f"Exported {count} records to backup.jsonl\n")
+
+    # Example 2: Export filtered data to JSON
+    print("=== Export Filtered Data ===")
+    count = client.export_to_json("python_docs.jsonl", filters={"category": "python"})
+    print(f"Exported {count} Python documents to python_docs.jsonl\n")
+
+    # Example 3: Export to CSV (without embeddings for readability)
+    print("=== Export to CSV ===")
+    count = client.export_to_csv("documents.csv", include_embeddings=False)
+    print(f"Exported {count} records to documents.csv\n")
+
+    # Example 4: Export to CSV with embeddings
+    print("=== Export to CSV with Embeddings ===")
+    count = client.export_to_csv("documents_full.csv", include_embeddings=True)
+    print(f"Exported {count} records with embeddings to documents_full.csv\n")
+
+    # Example 5: Backup and restore workflow
+    print("=== Backup and Restore Workflow ===")
+
+    # Backup
+    print("Creating backup...")
+    client.export_to_json("backup_full.jsonl")
+
+    # Simulate data loss
+    print("Simulating data loss...")
+    original_count = client.count()
+    for rowid in range(1, original_count + 1):
+        client.delete(rowid)
+    print(f"Records after deletion: {client.count()}")
+
+    # Restore
+    print("Restoring from backup...")
+    count = client.import_from_json("backup_full.jsonl")
+    print(f"Restored {count} records")
+    print(f"Records after restore: {client.count()}\n")
+
+    # Example 6: Data migration scenario
+    print("=== Data Migration Scenario ===")
+
+    # Export from source
+    print("Exporting from source database...")
+    client.export_to_json("migration.jsonl")
+
+    # Import to new database (simulated with same client)
+    print("Importing to destination database...")
+    # In real scenario, you would create a new client with different db_path
+    # new_client = SQLiteVecClient(table="documents", db_path="new.db")
+    # new_client.create_table(dim=128)
+    # new_client.import_from_json("migration.jsonl")
+    print("Migration complete!\n")
+
+    # Example 7: Filtered export for data sharing
+    print("=== Export Subset for Sharing ===")
+    count = client.export_to_csv(
+        "beginner_docs.csv", include_embeddings=False, filters={"level": "beginner"}
+    )
+    print(f"Exported {count} beginner-level documents for sharing\n")
+
+    # Cleanup
+    print("=== Cleanup ===")
+    import os
+
+    for file in [
+        "backup.jsonl",
+        "python_docs.jsonl",
+        "documents.csv",
+        "documents_full.csv",
+        "backup_full.jsonl",
+        "migration.jsonl",
+        "beginner_docs.csv",
+    ]:
+        if os.path.exists(file):
+            os.remove(file)
+            print(f"Removed {file}")
+
+    client.close()
+    print("\nExample complete!")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/sqlite_vec_client/base.py b/sqlite_vec_client/base.py
@@ -16,6 +16,7 @@
 
 import sqlite_vec
 
+from . import io as io_module
 from .exceptions import ConnectionError as VecConnectionError
 from .exceptions import TableNotFoundError
 from .logger import get_logger
@@ -635,6 +636,80 @@ def similarity_search_with_filter(
                 ) from e
             raise
 
+    def export_to_json(
+        self,
+        filepath: str,
+        include_embeddings: bool = True,
+        filters: dict[str, Any] | None = None,
+        batch_size: int = 1000,
+    ) -> int:
+        """Export records to JSON Lines format.
+
+        Args:
+            filepath: Path to output file
+            include_embeddings: Whether to include embeddings in export
+            filters: Optional metadata filters to apply
+            batch_size: Number of records to process at once
+
+        Returns:
+            Number of records exported
+        """
+        return io_module.export_to_json(
+            self, filepath, include_embeddings, filters, batch_size
+        )
+
+    def import_from_json(
+        self, filepath: str, skip_duplicates: bool = False, batch_size: int = 1000
+    ) -> int:
+        """Import records from JSON Lines format.
+
+        Args:
+            filepath: Path to input file
+            skip_duplicates: Whether to skip records with existing rowids
+            batch_size: Number of records to import at once
+
+        Returns:
+            Number of records imported
+        """
+        return io_module.import_from_json(self, filepath, skip_duplicates, batch_size)
+
+    def export_to_csv(
+        self,
+        filepath: str,
+        include_embeddings: bool = False,
+        filters: dict[str, Any] | None = None,
+        batch_size: int = 1000,
+    ) -> int:
+        """Export records to CSV format.
+
+        Args:
+            filepath: Path to output file
+            include_embeddings: Whether to include embeddings (as JSON string)
+            filters: Optional metadata filters to apply
+            batch_size: Number of records to process at once
+
+        Returns:
+            Number of records exported
+        """
+        return io_module.export_to_csv(
+            self, filepath, include_embeddings, filters, batch_size
+        )
+
+    def import_from_csv(
+        self, filepath: str, skip_duplicates: bool = False, batch_size: int = 1000
+    ) -> int:
+        """Import records from CSV format.
+
+        Args:
+            filepath: Path to input file
+            skip_duplicates: Whether to skip records with existing rowids
+            batch_size: Number of records to import at once
+
+        Returns:
+            Number of records imported
+        """
+        return io_module.import_from_csv(self, filepath, skip_duplicates, batch_size)
+
     @contextmanager
     def transaction(self) -> Generator[None, None, None]:
         """Context manager for atomic transactions.
diff --git a/sqlite_vec_client/io.py b/sqlite_vec_client/io.py
diff --git a/tests/test_io.py b/tests/test_io.py
