Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Performance regression checks: benchmarking tests #229

Merged
merged 11 commits into from
Nov 11, 2024
Merged

Performance regression checks: benchmarking tests #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
3b3b71b
Benchmarking regression tests
lcauser-oqc Oct 25, 2024
a9f8ea0
Added some testing tests
lcauser-oqc Oct 28, 2024
ffdb967
added clifford benchmark
lcauser-oqc Oct 28, 2024
3fad3d1
updated readme
lcauser-oqc Oct 30, 2024
a4178f1
deleted old files
lcauser-oqc Oct 31, 2024
3c9f3e0
added improvement test
lcauser-oqc Oct 31, 2024
9d193a5
updated readme
lcauser-oqc Nov 1, 2024
7ef9761
mistake in readme
lcauser-oqc Nov 1, 2024
3073081
Typo
lcauser-oqc Nov 1, 2024
659d582
added a mock live engine execution to benchmarks
lcauser-oqc Nov 4, 2024
215a6bd
poetry lock
lcauser-oqc Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/package-build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,4 +98,4 @@ jobs:
run: poetry install --sync

- name: Unit tests
run: poetry run pytest
run: poetry run pytest --benchmark-skip
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ calibration_results/
**/*.temp
**/tmp.*

# Benchmarks
.benchmarks/

# Docs
docs/build/*
docs/source/*
Expand Down
13 changes: 13 additions & 0 deletions README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,19 @@ Additionally, to report any concerns or

|
----------------------
Benchmarking
----------------------

The performance of QAT can be measured using our pre-defined benchmarks: :code:`poetry run pytest --benchmark-only`.
To compare to main, checkout the main branch and run :code:`poetry run pytest --benchmark-only --benchmark-save="<benchmark-name>"`.
Then checkout back to the branch you are working and run :code:`poetry run pytest --benchmark-only --benchmark-save="<benchmark-name>" --benchmark-compare --benchmark-compare-fail=min:50%`.
If the test fails, it might indicate a performance regression: use the comparison table that is outputted to verify.
The performance of pull requests to main will be automatically tested.
See the `pytest-benchmark <https://pytest-benchmark.readthedocs.io/en/latest/usage.html>`_ documentation for more information on how to use it.

|
----------------------
FAQ
----------------------
Expand Down
1,505 changes: 751 additions & 754 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ autoflake = "^2.2.0"
isort = "^5.13.2"
docplex = "^2.21.207"
pre-commit = "^3.2.0"
pytest-benchmark = "^4.0.0"

[tool.poetry.group.licenses]
optional = true
Expand Down Expand Up @@ -136,4 +137,4 @@ format-code = "poetry_scripts:format_code"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"
priority = "primary"
Empty file added tests/benchmarking/__init__.py
View file
Open in desktop
Empty file.
15 changes: 15 additions & 0 deletions tests/benchmarking/qasm/10qb_ghz.qasm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[10];
creg c[10];
h q[0];
cx q[0], q[1];
cx q[1], q[2];
cx q[2], q[3];
cx q[3], q[4];
cx q[4], q[5];
cx q[5], q[6];
cx q[6], q[7];
cx q[7], q[8];
cx q[8], q[9];
measure q -> c;
160 changes: 160 additions & 0 deletions tests/benchmarking/qasm/10qb_random_cnot.qasm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[10];
creg c[10];
u3(2.1476154995782335, 3.8199553784163194, 4.2454916175770645) q[0];
u3(4.050215034506989, 4.760568095907491, 3.396091016455767) q[1];
u3(4.386798875432581, 2.64391401939085, 4.641209421796389) q[2];
u3(6.188669264630735, 1.5232788292469255, 3.5987642076340554) q[3];
u3(5.091820489910071, 4.55352107708396, 5.3092933432957485) q[4];
u3(5.2501596312696055, 1.115614436416206, 1.2773565665236108) q[5];
u3(0.8159055856276346, 3.112855881632552, 0.045396816840497234) q[6];
u3(5.424173744031299, 3.9670950484042677, 3.6761161503095257) q[7];
u3(0.5465152234834442, 5.910844912794045, 0.5812240803011206) q[8];
u3(2.0187305348565685, 6.16030364236002, 2.4360975620535594) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(0.8123125007312298, 0.6148585070167231, 0.736178782135525) q[0];
u3(0.7659003651527424, 0.37631704599906773, 0.6213598995356535) q[1];
u3(3.0287212326490884, 0.9626311703384103, 4.3720082182605005) q[2];
u3(2.3548897357774825, 1.860572080686291, 4.255155410793541) q[3];
u3(1.071948354056817, 4.385335985468801, 5.0688745154886465) q[4];
u3(0.1032826341991083, 0.38305043965672037, 3.5132076404261845) q[5];
u3(3.6192712801930718, 3.283590670591691, 1.1168060117763603) q[6];
u3(0.5386542989000254, 1.0780695053112679, 5.920418397929598) q[7];
u3(4.616823999533406, 5.643020446897611, 1.7324997416884562) q[8];
u3(4.590248277985277, 4.9316630926502025, 1.1061214017930179) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(6.146355263634162, 1.8999513184139087, 4.140355349266817) q[0];
u3(4.906129195299491, 3.5386500090489315, 2.786243507890957) q[1];
u3(0.6332830638217943, 0.8080387767603451, 5.61453144838349) q[2];
u3(2.0265652535084135, 1.3290438198905894, 2.854055432954448) q[3];
u3(1.214652215013949, 4.651629128387061, 3.6946059819323187) q[4];
u3(2.5204389853308142, 4.262259888043107, 0.9711069088036576) q[5];
u3(4.36953863846858, 5.679293679068043, 1.0594444892973758) q[6];
u3(5.699848450751837, 0.9082555675439579, 2.8830377987325586) q[7];
u3(2.087886534654373, 4.7096739732597666, 5.079764263317569) q[8];
u3(1.1734692712643875, 3.8881137321160946, 0.9838633326741538) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(2.3740805458166356, 2.4029591583281475, 0.9264585610627256) q[0];
u3(6.067304332958309, 0.7616606594247852, 2.4431081228394422) q[1];
u3(2.960847198635117, 4.061271917629743, 4.748873349452179) q[2];
u3(1.7782683551480225, 3.6497020269990212, 1.6071935625390077) q[3];
u3(2.051994990919352, 6.200262081428702, 4.77979310091581) q[4];
u3(2.8951789242569568, 1.3338407285774465, 3.723042367618023) q[5];
u3(2.429195063286592, 1.0847451099950298, 6.279793015219683) q[6];
u3(2.3574921290684263, 0.8618573915600358, 0.7776453195158278) q[7];
u3(1.174901194944264, 3.977845955068551, 0.8392630346355411) q[8];
u3(2.309645003060026, 3.669075644631806, 1.1475858700376442) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(4.208208423347275, 0.44816107770742314, 0.17658686915098662) q[0];
u3(3.4373975068726317, 0.1793550902470097, 0.16944209301730268) q[1];
u3(3.3456844156739964, 3.3529347180608506, 0.8860579033639527) q[2];
u3(0.6891308281239467, 0.3975281392651653, 4.394096046794606) q[3];
u3(5.2908557014895745, 4.852062048472861, 5.973771579580277) q[4];
u3(2.1698269790851628, 5.84268546572853, 1.3697216809734303) q[5];
u3(3.7828064618099684, 1.8454046390083347, 3.0422440207823356) q[6];
u3(0.5191331897614178, 0.1383746265796521, 5.323959052628703) q[7];
u3(6.022428841569019, 5.6878832191083735, 0.5366042001812283) q[8];
u3(2.998161287591667, 2.9341129601556086, 5.365576467493857) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(4.2792532385518465, 0.894384316103267, 4.193079822308461) q[0];
u3(3.7053658702424195, 2.4414672438270344, 6.122309061250553) q[1];
u3(2.060756357985279, 2.384081686664948, 5.757840289977498) q[2];
u3(2.979885458783252, 4.019500862667376, 5.657160320507938) q[3];
u3(0.9250294211968126, 3.496146387987904, 2.6521585347834273) q[4];
u3(6.034403602281088, 5.058571186352238, 2.7662121247866693) q[5];
u3(5.636283759966305, 1.4370592490921679, 0.315001508528092) q[6];
u3(2.9966852179494703, 5.586758879161887, 2.1719942215981107) q[7];
u3(2.417281132904969, 2.920986433111673, 5.442531735681578) q[8];
u3(5.41136419070043, 6.007338996139764, 1.5049790947891035) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(1.7023252396545367, 1.9478277883510777, 3.7612624863801885) q[0];
u3(1.3791631757063214, 3.555088714605354, 2.1381067388331894) q[1];
u3(1.0549623721801304, 2.3081802603833235, 1.7649096051373347) q[2];
u3(3.08985673105197, 1.3611914784707968, 5.058618478105476) q[3];
u3(1.186425672751201, 4.947775678749102, 1.7106267514699436) q[4];
u3(0.2777347545847968, 1.7377284408840898, 2.6649728231716017) q[5];
u3(3.3694436899699918, 4.879476043461998, 3.8767038389527166) q[6];
u3(6.222063562880843, 2.8587386021573886, 0.41863481802587604) q[7];
u3(1.2705682887507463, 1.378397408994558, 2.684218142207635) q[8];
u3(2.6729994642440182, 4.557845767622726, 4.307520241147747) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(4.100123836325104, 5.022873542927932, 3.2667802215994484) q[0];
u3(0.028587224951316596, 2.070721842216032, 5.441036586555664) q[1];
u3(2.367014332211276, 2.0076359530127075, 2.961320395665276) q[2];
u3(2.8098289719214384, 5.676971378883951, 2.125915022677554) q[3];
u3(1.4220484049078315, 2.1748026183219107, 2.0444745734014247) q[4];
u3(2.0847493568038016, 0.6880625313901045, 3.7645129853317725) q[5];
u3(5.752479239420571, 1.321165864641857, 0.8045918772660725) q[6];
u3(2.8876429865552002, 4.228553816959603, 4.380084516121471) q[7];
u3(5.970710235368468, 0.33358964932408525, 5.997145881049767) q[8];
u3(5.858111162898702, 1.1881820771053189, 5.2159468150357275) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(2.0128820703743537, 2.5490107473433534, 0.8636461222743614) q[0];
u3(0.21518649972015771, 2.0768366561032545, 0.7383645523431643) q[1];
u3(5.506620884435293, 6.008115491486742, 2.3630874422671955) q[2];
u3(4.2799588832852535, 2.228753936083146, 2.3984018539348693) q[3];
u3(4.703895803505276, 1.2432363321460245, 3.256798195471329) q[4];
u3(2.0516250087859556, 4.063325153592703, 1.5659145082601382) q[5];
u3(5.731311489439932, 3.971757328214825, 4.217152990704095) q[6];
u3(5.776907807401078, 0.9526679512736421, 2.14271080135232) q[7];
u3(3.938897024600601, 5.92895250274184, 1.0905368642061466) q[8];
u3(2.1985111660832235, 2.0839506701568915, 5.540189268629577) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(0.12796584980929313, 5.826913214957174, 2.7155071351451276) q[0];
u3(5.756491129158667, 4.8030432559759975, 4.8576917393240695) q[1];
u3(0.7649831042720576, 5.47203513006738, 0.7096207234473794) q[2];
u3(3.7899238968036735, 5.852289804611468, 0.2777690187744513) q[3];
u3(2.5041425413541387, 1.5770973295757982, 6.125344218237444) q[4];
u3(3.705870575034715, 6.040125500880039, 2.760153129150555) q[5];
u3(0.7441818812294555, 4.231228485233901, 6.2813647792398575) q[6];
u3(5.221783683598857, 2.2955817305033346, 1.4787114225665745) q[7];
u3(3.237672276846762, 0.6536479042046223, 0.12725564980679163) q[8];
u3(2.5464418661234167, 1.087589107809967, 3.87116512335739) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(4.506939621598436, 0.5274988381397553, 6.237201066255194) q[0];
u3(0.9916102148500096, 0.5632238245218942, 3.6191342928509647) q[1];
u3(3.3029262476713885, 5.271532983257014, 2.9899958917558664) q[2];
u3(0.10108438587549094, 1.0216049899008273, 5.6769838692552135) q[3];
u3(3.807222096043554, 3.7243918019820113, 0.8943134830623742) q[4];
u3(3.509084223409552, 1.4239459680268831, 4.42268381675587) q[5];
u3(0.5854954719853341, 1.8346266000931504, 3.5712194229497247) q[6];
u3(0.8259591927508211, 0.7904851976172592, 2.7642246833611175) q[7];
u3(5.692561103875125, 5.2303069022442905, 1.5362328399346596) q[8];
u3(0.9448399469329687, 5.089689938085127, 4.762931686091827) q[9];
measure q -> c;
69 changes: 69 additions & 0 deletions tests/benchmarking/qasm/2qb_clifford.qasm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
OPENQASM 2.0;
include "qelib1.inc";
gate gate_Clifford_2Q_5451_ q0,q1 { h q1; h q0; s q0; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_5896_ q0,q1 { h q0; cx q0,q1; h q0; s q0; sdg q1; h q1; y q0; }
gate gate_Clifford_2Q_8699_ q0,q1 { h q0; h q1; cx q0,q1; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_10949_ q0,q1 { h q0; h q1; h q0; s q0; sdg q1; h q1; cx q0,q1; sdg q0; h q0; x q0; x q1; }
gate gate_Clifford_2Q_401_ q0,q1 { sdg q1; h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; x q1; }
gate gate_Clifford_2Q_1660_ q0,q1 { sdg q0; h q0; h q1; s q1; cx q0,q1; h q1; s q1; z q0; }
gate gate_Clifford_2Q_9480_ q0,q1 { h q0; h q1; h q1; s q1; cx q0,q1; cx q1,q0; h q1; s q1; y q0; }
gate gate_Clifford_2Q_10928_ q0,q1 { h q0; h q1; h q0; s q0; sdg q1; h q1; cx q0,q1; h q1; s q1; }
gate gate_Clifford_2Q_2871_ q0,q1 { h q0; s q0; h q1; s q1; cx q0,q1; cx q1,q0; cx q0,q1; x q0; z q1; }
gate gate_Clifford_2Q_3592_ q0,q1 { h q1; h q1; s q1; cx q0,q1; sdg q0; h q0; y q0; }
gate gate_Clifford_2Q_10011_ q0,q1 { h q0; h q1; sdg q0; h q0; sdg q1; h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; y q0; z q1; }
gate gate_Clifford_2Q_4876_ q0,q1 { h q1; h q0; s q0; cx q0,q1; sdg q0; h q0; z q0; }
gate gate_Clifford_2Q_3146_ q0,q1 { h q1; cx q0,q1; cx q1,q0; h q0; s q0; y q0; y q1; }
gate gate_Clifford_2Q_9535_ q0,q1 { h q0; h q1; h q1; s q1; cx q0,q1; cx q1,q0; sdg q0; h q0; h q1; s q1; z q0; z q1; }
gate gate_Clifford_2Q_2960_ q0,q1 { h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; }
gate gate_Clifford_2Q_4713_ q0,q1 { h q1; sdg q0; h q0; h q1; s q1; cx q0,q1; cx q1,q0; sdg q0; h q0; sdg q1; h q1; y q0; x q1; }
gate gate_Clifford_2Q_7416_ q0,q1 { h q0; sdg q0; h q0; h q1; s q1; cx q0,q1; h q1; s q1; y q0; }
gate gate_Clifford_2Q_6331_ q0,q1 { h q0; sdg q1; h q1; cx q0,q1; cx q1,q0; sdg q0; h q0; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_987_ q0,q1 { sdg q0; h q0; cx q0,q1; y q0; z q1; }
gate gate_Clifford_2Q_317_ q0,q1 { cx q0,q1; cx q1,q0; cx q0,q1; z q0; x q1; }
gate gate_Clifford_2Q_976_ q0,q1 { sdg q0; h q0; cx q0,q1; }
qreg q[2];
creg meas[2];
gate_Clifford_2Q_5451_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_5896_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_8699_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10949_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_401_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_1660_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_9480_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10928_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_2871_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_3592_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10011_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_4876_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_3146_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_9535_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_2960_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_4713_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_7416_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_6331_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_987_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_317_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_976_ q[0],q[1];
barrier q[0],q[1];
measure q[0] -> meas[0];
measure q[1] -> meas[1];
37 changes: 37 additions & 0 deletions tests/benchmarking/qasm/2qb_random_cnot.qasm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[2];
creg c[2];
u3(1.3006602139645063, 0.534181202485765, 5.689257839837927) q[0];
u3(0.6435923222995911, 2.6357056328036563, 4.583407399749809) q[1];
cx q[0], q[1];
u3(0.5469197324369987, 5.838208081346666, 4.8559930708683225) q[0];
u3(2.1773336422972918, 1.463075857751379, 1.7386307630262692) q[1];
cx q[0], q[1];
u3(0.08906857576487162, 2.0524090814689617, 3.924823508136547) q[0];
u3(5.691523965972233, 4.745449389658818, 3.618032619773981) q[1];
cx q[0], q[1];
u3(1.6052519941473888, 1.7146648534578364, 2.7581683498790937) q[0];
u3(3.695617942720016, 1.4719988957904238, 4.722941564499853) q[1];
cx q[0], q[1];
u3(5.593711172292128, 4.14687486034386, 3.047863806461297) q[0];
u3(2.3259486973738075, 1.8055595481953204, 2.830280329583103) q[1];
cx q[0], q[1];
u3(3.0508523408675328, 5.782789478403682, 5.782316802038208) q[0];
u3(5.309221711611476, 3.59325273719956, 0.7510463983108712) q[1];
cx q[0], q[1];
u3(2.577591983152861, 4.873374951287487, 1.1955443415738125) q[0];
u3(2.7029681627604765, 0.5410471721128901, 2.164425117559505) q[1];
cx q[0], q[1];
u3(1.580605669989407, 1.0969371557584393, 4.2352542755153175) q[0];
u3(5.29713698252154, 0.8143890512769186, 4.535540282844311) q[1];
cx q[0], q[1];
u3(2.98211362929654, 0.6985239567305469, 4.271335180649374) q[0];
u3(0.3828013976141185, 6.142020345273884, 2.9774674667682075) q[1];
cx q[0], q[1];
u3(3.286579382426756, 3.3449235001761486, 2.1157006056982524) q[0];
u3(3.6113600392886864, 4.34759595156976, 2.9588514547434657) q[1];
cx q[0], q[1];
u3(0.4285650460830407, 1.5673997888781974, 5.446012275807939) q[0];
u3(5.312063183960821, 4.807739604050404, 5.4774519419546515) q[1];
measure q -> c;
7 changes: 7 additions & 0 deletions tests/benchmarking/qasm/bell_state.qasm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[2];
creg c[2];
h q[0];
cx q[0], q[1];
measure q -> c;
Loading
Loading