Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 65 additions & 24 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,38 +7,79 @@ on:
branches: [ master ]

jobs:
test:
test-core:
name: Core profile (no CUDA extension)
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.8, 3.9, "3.10"]
python-version: ["3.10", "3.11", "3.12"]

steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install pytest

- name: Run tests
run: |
pytest tests/
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install core + dev dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements-dev.txt
pip install -e . --no-build-isolation

- name: Run core protocol tests (CPU-only)
run: |
pytest --noconftest tests/test_protocol.py -q

test-gpu-profile:
name: GPU dependency profile (import smoke)
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install GPU profile dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements-gpu.txt

- name: Validate GPU profile imports
run: |
python -c "import torch; print('torch:', torch.__version__)"
python -c "import cuda; print('cuda-python import: ok')"

test-ray-profile:
name: Ray profile (import smoke)
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install Ray profile dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements-ray.txt

- name: Validate Ray profile imports
run: |
python -c "import ray; print('ray:', ray.__version__)"

build:
runs-on: ubuntu-latest
needs: test
needs: [test-core, test-gpu-profile, test-ray-profile]
if: github.ref == 'refs/heads/master'

steps:
- uses: actions/checkout@v3
- name: Build and publish Docker image
run: |
# Сборка Docker-образа
docker build -f deployment/docker/Dockerfile.unified -t zerolink:${{ github.sha }} .
docker tag zerolink:${{ github.sha }} zerolink:latest
- uses: actions/checkout@v4
- name: Build and publish Docker image
run: |
docker build -f deployment/docker/Dockerfile.unified -t zerolink:${{ github.sha }} .
docker tag zerolink:${{ github.sha }} zerolink:latest
22 changes: 18 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,18 +44,32 @@ ZeroLink v2.0 обеспечивает значительный прирост

```bash
git clone <repo-url>
cd pynexus_rex_v2
cd zerolink

# Установка зависимостей
# Профили зависимостей
# core (минимальный профиль)
pip install -r requirements.txt

# Сборка C++ Extension
# gpu (PyTorch + CUDA Python)
pip install -r requirements-gpu.txt

# ray (distributed профиль)
pip install -r requirements-ray.txt

# dev (инструменты разработки)
pip install -r requirements-dev.txt

# Сборка C++ Extension (опционально, для GPU пути)
python setup.py build_ext --inplace
```

### 2. Запуск тестов

```bash
# CPU-only профиль (без conftest GPU импортов)
pytest --noconftest tests/test_protocol.py -q

# Полный прогон (требует GPU профиль и torch/cuda)
pytest -q
```

Expand All @@ -79,7 +93,7 @@ runtime.start() # Запускает сервер в фоне
```python
from zerolink.workers import GPUWorker

worker = GPUWorker(sock_path="/tmp/pynexus.sock", device_id=0)
worker = GPUWorker(sock_path="/tmp/zerolink.sock", device_id=0)
worker.connect()

# Event loop
Expand Down
20 changes: 10 additions & 10 deletions docs/performance_estimation.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,35 +25,35 @@ def realistic_performance_estimation():
metrics = {
"IPC Bandwidth": {
"traditional": "~100-500 MB/s (через pickle)",
"pynexus": ">850 MB/s (zero-copy)",
"zerolink": ">850 MB/s (zero-copy)",
"improvement": "2-10x быстрее"
},
"Allocation Speed": {
"traditional": "O(log n) - зависит от состояния пула",
"pynexus": "O(1) - через Buddy Allocator",
"zerolink": "O(1) - через Buddy Allocator",
"improvement": "2-5x быстрее"
},
"Memory Efficiency": {
"traditional": "Высокое потребление (дублирование тензоров)",
"pynexus": "Минимизированное (zero-copy sharing)",
"zerolink": "Минимизированное (zero-copy sharing)",
"improvement": "2-5x меньше памяти"
},
"Fragmentation": {
"traditional": "Высокая (особенно при длительных сессиях)",
"pynexus": "Минимальная (Buddy Allocator + defragmentation)",
"zerolink": "Минимальная (Buddy Allocator + defragmentation)",
"improvement": "5-10x меньше фрагментации"
},
"Latency": {
"traditional": "Высокая (сериализация + копирование)",
"pynexus": "Низкая (прямой доступ к памяти)",
"zerolink": "Низкая (прямой доступ к памяти)",
"improvement": "10-30% снижение"
}
}

for metric, values in metrics.items():
print(f"{metric}:")
print(f" Традиционный подход: {values['traditional']}")
print(f" ZeroLink: {values['pynexus']}")
print(f" ZeroLink: {values['zerolink']}")
print(f" Улучшение: {values['improvement']}\n")

# Сценарии использования и потенциальный прирост
Expand Down Expand Up @@ -118,25 +118,25 @@ def realistic_performance_estimation():
"Standard PyTorch": {
"pros": "Простота, зрелость",
"cons": "Высокое потребление памяти, фрагментация",
"vs_pynexus": "ZeroLink обеспечивает 2-5x более эффективное использование памяти"
"vs_zerolink": "ZeroLink обеспечивает 2-5x более эффективное использование памяти"
},
"Custom CUDA Kernels": {
"pros": "Максимальная производительность для специфичных задач",
"cons": "Высокая сложность разработки и поддержки",
"vs_pynexus": "ZeroLink предоставляет универсальное решение с меньшей сложностью"
"vs_zerolink": "ZeroLink предоставляет универсальное решение с меньшей сложностью"
},
"NCCL": {
"pros": "Высокая производительность для collective операций",
"cons": "Ограниченная применимость (не для произвольного IPC)",
"vs_pynexus": "ZeroLink дополняет NCCL, обеспечивая эффективный point-to-point IPC"
"vs_zerolink": "ZeroLink дополняет NCCL, обеспечивая эффективный point-to-point IPC"
}
}

for solution, details in comparison.items():
print(f"{solution}:")
print(f" Плюсы: {details['pros']}")
print(f" Минусы: {details['cons']}")
print(f" PyNexus vs: {details['vs_pynexus']}\n")
print(f" ZeroLink vs: {details['vs_zerolink']}\n")

print("Заключение:")
print("ZeroLink v2.0 обеспечивает значительный прирост производительности")
Expand Down
35 changes: 27 additions & 8 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[build-system]
requires = ["setuptools>=61.0.0", "wheel", "torch"]
requires = ["setuptools>=61.0.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
Expand Down Expand Up @@ -27,24 +27,43 @@ classifiers = [
]

dependencies = [
"torch>=1.12.0",
"cuda-python>=11.7",
"blake3>=0.4.0",
"numpy>=1.24.0",
]

[project.optional-dependencies]
gpu = [
"torch>=1.12.0",
"cuda-python>=11.7",
]
ray = [
"ray>=2.0.0",
]
cgpu = [
"cgpu>=0.1.0",
]
dev = [
"pytest>=7.0",
"pytest-cov",
"black",
"mypy",
]
full = [
"torch>=1.12.0",
"cuda-python>=11.7",
"ray>=2.0.0",
"cgpu>=0.1.0",
"pytest>=7.0",
"pytest-cov",
"black",
"mypy",
]

[project.urls]
Homepage = "https://github.com/your-org/pynexus-rex"
Documentation = "https://github.com/your-org/pynexus-rex/docs"
Repository = "https://github.com/your-org/pynexus-rex.git"
"Bug Tracker" = "https://github.com/your-org/pynexus-rex/issues"
Homepage = "https://github.com/your-org/zerolink"
Documentation = "https://github.com/your-org/zerolink/docs"
Repository = "https://github.com/your-org/zerolink.git"
"Bug Tracker" = "https://github.com/your-org/zerolink/issues"

[tool.setuptools]
packages = ["pynexus_rex"]
packages = ["zerolink"]
3 changes: 3 additions & 0 deletions requirements-cgpu.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Optional low-level CUDA driver integration
-r requirements-gpu.txt
cgpu>=0.1.0
6 changes: 6 additions & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Development/test tools
-r requirements.txt
pytest>=7.0
pytest-cov
black
mypy
4 changes: 4 additions & 0 deletions requirements-gpu.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# GPU profile
-r requirements.txt
torch>=1.12.0
cuda-python>=11.7
3 changes: 3 additions & 0 deletions requirements-ray.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Distributed Ray profile
-r requirements-gpu.txt
ray>=2.0.0
6 changes: 2 additions & 4 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
torch>=1.12.0
cuda-python>=11.7
# Core runtime dependencies (CPU/control-plane friendly)
blake3>=0.4.0
ray>=2.0.0
cgpu>=0.1.0
numpy>=1.24.0
Loading
Loading