Skip to content

Hualxie/sd intel #261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
xieofxie wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

Hualxie/sd intel #261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1a0502e
ov
Mar 13, 2026
0fcaf0e
bug fix
Mar 13, 2026
62b5d6a
add 768
Mar 16, 2026
6f04f83
reuse_cache false
Mar 16, 2026
da4a0e4
bug fix
Mar 16, 2026
878b205
align
Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .aitk/configs/checks.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
"modelProjectCheck": 42,
"oliveCheck": 60,
"oliveJsonCheck": 155,
"pathCheck": 1293,
"pathCheck": 1289,
"requirementsCheck": 37,
"templateCheck": 3,
"venvRequirementsCheck": 15
"venvRequirementsCheck": 16
}
4 changes: 3 additions & 1 deletion .aitk/configs/model_list.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,9 @@
"modelLink": "https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5",
"id": "huggingface/stable-diffusion-v1-5/stable-diffusion-v1-5",
"runtimes": [
"QNN"
"IntelCPU",
"IntelGPU",
"IntelNPU"
],
"architecture": "Transformer",
"status": "Hide",
Expand Down
3 changes: 3 additions & 0 deletions .aitk/requirements/requirements-IntelNPU-SD.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
accelerate==1.12.0
diffusers==0.35.0
torch-fidelity==0.3.0
18 changes: 18 additions & 0 deletions sd-legacy-stable-diffusion-v1-5/aitk/config_text_encoder.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,24 @@
"example_input_func": "text_encoder_conversion_inputs",
"output_model": "text_encoder"
},
"ov_io_update": {
"type": "OpenVINOIoUpdate",
"static": true,
"input_shapes": [
[
1,
77
]
],
"input_names": [ "input_ids" ],
"reuse_cache": false
},
"ov_encapsulation": {
"type": "OpenVINOEncapsulation",
"target_device": "npu",
"ov_version": "2025.1",
"reuse_cache": false
},
"optimize": {
"type": "OrtTransformersOptimization",
"model_type": "clip",
Expand Down
21 changes: 21 additions & 0 deletions sd-legacy-stable-diffusion-v1-5/aitk/config_unet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,27 @@
"example_input_func": "get_unet_ov_example_input",
"output_model": "unet"
},
"ov_io_update": {
"type": "OpenVINOIoUpdate",
"static": true,
"input_shapes": [
[
1, 4, 64, 64
],
[ ],
Copy link
Contributor Author

@xieofxie xieofxie Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

result in a single float input, not 1 dimension float array

[ 1, 77, 768 ]
],
"input_names": [
"sample", "timestep", "encoder_hidden_states"
],
"reuse_cache": false
},
"ov_encapsulation": {
"type": "OpenVINOEncapsulation",
"target_device": "npu",
"ov_version": "2025.1",
"reuse_cache": false
},
"optimize": {
"type": "OrtTransformersOptimization",
"model_type": "unet",
Expand Down
17 changes: 17 additions & 0 deletions sd-legacy-stable-diffusion-v1-5/aitk/config_vae_decoder.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,23 @@
"example_input_func": "vae_decoder_conversion_inputs",
"output_model": "vae_decoder"
},
"ov_io_update": {
"type": "OpenVINOIoUpdate",
"static": true,
"input_shapes": [
[ 1, 4, 64, 64 ]
],
"input_names": [
"latent_sample"
],
"reuse_cache": false
},
"ov_encapsulation": {
"type": "OpenVINOEncapsulation",
"target_device": "npu",
"ov_version": "2025.1",
"reuse_cache": false
},
"optimize": {
"type": "OrtTransformersOptimization",
"model_type": "vae",
Expand Down
17 changes: 17 additions & 0 deletions sd-legacy-stable-diffusion-v1-5/aitk/config_vae_encoder.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,23 @@
"example_input_func": "vae_encoder_conversion_inputs",
"output_model": "vae_encoder"
},
"ov_io_update": {
"type": "OpenVINOIoUpdate",
"static": true,
"input_shapes": [
[ 1, 3, 512, 512]
],
"input_names": [
"sample"
],
"reuse_cache": false
},
"ov_encapsulation": {
"type": "OpenVINOEncapsulation",
"target_device": "npu",
"ov_version": "2025.1",
"reuse_cache": false
},
"optimize": {
"type": "OrtTransformersOptimization",
"model_type": "vae",
Expand Down
2 changes: 1 addition & 1 deletion sd-legacy-stable-diffusion-v1-5/aitk/inference_sample.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"source": [
"model_dir = \"./model/optimized/stable-diffusion-v1-5/stable-diffusion-v1-5\"\n",
"\n",
"ExecutionProvider=\"QNNExecutionProvider\""
"ExecutionProvider=\"OpenVINOExecutionProvider\""
]
},
{
Expand Down
7 changes: 5 additions & 2 deletions sd-legacy-stable-diffusion-v1-5/aitk/info.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ keywords:
arch: sd
recipes:
- file: "sd_qnn_workflow.json"
device: npu
ep: QNNExecutionProvider
devices:
- npu
- cpu
- gpu
ep: OpenVINOExecutionProvider
aitk:
modelInfo:
id: "huggingface/stable-diffusion-v1-5/stable-diffusion-v1-5"
Expand Down
2 changes: 1 addition & 1 deletion sd-legacy-stable-diffusion-v1-5/aitk/sd_qnn_workflow.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
{
"device": "npu",
"execution_providers": [
"QNNExecutionProvider"
"OpenVINOExecutionProvider"
]
}
]
Expand Down
38 changes: 7 additions & 31 deletions sd-legacy-stable-diffusion-v1-5/aitk/sd_qnn_workflow.json.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,39 +19,15 @@
"name": "Evaluate on",
"type": "enum",
"displayNames": [
"Qualcomm NPU",
"CPU"
"Intel CPU",
"Intel NPU",
"Intel GPU"
],
"path": "systems.target_system.accelerators.0.execution_providers.0",
"path": "systems.target_system.accelerators.0.device",
"values": [
"QNNExecutionProvider",
"CPUExecutionProvider"
],
"actions": [
[
{
"type": "update",
"path": "systems.target_system.accelerators.0.execution_providers.0",
"value": "QNNExecutionProvider"
},
{
"type": "update",
"path": "systems.target_system.accelerators.0.device",
"value": "npu"
}
],
[
{
"type": "update",
"path": "systems.target_system.accelerators.0.execution_providers.0",
"value": "CPUExecutionProvider"
},
{
"type": "update",
"path": "systems.target_system.accelerators.0.device",
"value": "cpu"
}
]
"cpu",
"npu",
"gpu"
],
"readOnly": false
},
Expand Down
47 changes: 23 additions & 24 deletions sd-legacy-stable-diffusion-v1-5/aitk/sd_qnn_workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,35 +96,34 @@ def main():
config_name = f"config_{submodel_name}.json"
copy_olive_config(history_folder, config_name, cache_dir, output_dir, activation_type, precision)

# run stable_diffusion.py to generate onnx unoptimized model
subprocess.run([sys.executable, "stable_diffusion.py",
"--script_dir", history_folder,
"--model_id", "stable-diffusion-v1-5/stable-diffusion-v1-5",
"--provider", "cpu",
"--format", "qdq",
"--optimize",
"--only_conversion"],
check=True)

# # run evaluation.py to generate data
subprocess.run([sys.executable, "evaluation.py",
"--script_dir", history_folder,
"--save_data",
"--model_id", "stable-diffusion-v1-5/stable-diffusion-v1-5",
"--num_inference_steps", "25",
"--seed", "0",
"--dataset_name", dataset_name,
"--dataset_split", dataset_split,
"--num_data", str(num_data),
"--guidance_scale", "7.5"],
check=True)
# # run stable_diffusion.py to generate onnx unoptimized model
# subprocess.run([sys.executable, "stable_diffusion.py",
# "--script_dir", history_folder,
# "--model_id", "stable-diffusion-v1-5/stable-diffusion-v1-5",
# "--provider", "cpu",
# "--format", "qdq",
# "--optimize",
# "--only_conversion"],
# check=True)

# # # run evaluation.py to generate data
# subprocess.run([sys.executable, "evaluation.py",
# "--script_dir", history_folder,
# "--save_data",
# "--model_id", "stable-diffusion-v1-5/stable-diffusion-v1-5",
# "--num_inference_steps", "25",
# "--seed", "0",
# "--dataset_name", dataset_name,
# "--dataset_split", dataset_split,
# "--num_data", str(num_data),
# "--guidance_scale", "7.5"],
# check=True)

# run stable_diffusion.py to generate onnx quantized model
subprocess.run([sys.executable, "stable_diffusion.py",
"--script_dir", history_folder,
"--model_id", "stable-diffusion-v1-5/stable-diffusion-v1-5",
"--provider", "cpu",
"--format", "qdq",
"--provider", "openvino",
"--optimize"],
check=True)

Expand Down
18 changes: 17 additions & 1 deletion sd-legacy-stable-diffusion-v1-5/aitk/sd_utils/onnx_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@


class PatchedOnnxRuntimeModel(OnnxRuntimeModel):
def __init__(self, model=None, **kwargs):
def __init__(self, model=None, is_ov_save=False, **kwargs):
self.model = model
self.is_ov_save = is_ov_save
self.model_save_dir = kwargs.get("model_save_dir", None)
self.latencies = []

Expand All @@ -34,6 +35,17 @@ def load_model(path: Union[str, Path], provider=None, sess_options=None, provide
return ort.InferenceSession(path, sess_options=sess_options)

def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs):
if self.is_ov_save:
patterns = ("*.onnx", "*.bin", "*.xml")
for pattern in patterns:
for file in self.model.glob(pattern):
dst_path = Path(save_directory).joinpath(file.name)
try:
shutil.copyfile(file, dst_path)
except shutil.SameFileError:
pass
return

model_file_name = ONNX_WEIGHTS_NAME

src_path = self.model_save_dir.joinpath(model_file_name)
Expand Down Expand Up @@ -73,8 +85,12 @@ def _from_pretrained(
model_id: Union[str, Path],
provider: Optional[str] = None,
sess_options: Optional["ort.SessionOptions"] = None,
is_ov_save: bool = False,
**kwargs,
):
if is_ov_save:
return cls(model=Path(model_id), is_ov_save=True)

model_file_name = ONNX_WEIGHTS_NAME
model_path = Path(model_id, model_file_name).as_posix()

Expand Down
Loading
Loading