Skip to content

Commit

Permalink
Merge pull request #229 from oqc-community/feature/lc/benchmarking_tests
Browse files Browse the repository at this point in the history 
Performance regression checks: benchmarking tests
  • Loading branch information
@lcauser-oqc
lcauser-oqc authored Nov 11, 2024
2 parents 164afc9 + 215a6bd commit de1ab83
Show file tree
Hide file tree
Showing 13 changed files with 1,156 additions and 756 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/package-build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,4 +98,4 @@ jobs:
run: poetry install --sync

- name: Unit tests
run: poetry run pytest
run: poetry run pytest --benchmark-skip
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ calibration_results/
**/*.temp
**/tmp.*

# Benchmarks
.benchmarks/

# Docs
docs/build/*
docs/source/*
Expand Down
13 changes: 13 additions & 0 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,19 @@ Additionally, to report any concerns or

|
----------------------
Benchmarking
----------------------

The performance of QAT can be measured using our pre-defined benchmarks: :code:`poetry run pytest --benchmark-only`.
To compare to main, checkout the main branch and run :code:`poetry run pytest --benchmark-only --benchmark-save="<benchmark-name>"`.
Then checkout back to the branch you are working and run :code:`poetry run pytest --benchmark-only --benchmark-save="<benchmark-name>" --benchmark-compare --benchmark-compare-fail=min:50%`.
If the test fails, it might indicate a performance regression: use the comparison table that is outputted to verify.
The performance of pull requests to main will be automatically tested.
See the `pytest-benchmark <https://pytest-benchmark.readthedocs.io/en/latest/usage.html>`_ documentation for more information on how to use it.

|
----------------------
FAQ
----------------------
Expand Down
1,505 changes: 751 additions & 754 deletions poetry.lock
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ autoflake = "^2.2.0"
isort = "^5.13.2"
docplex = "^2.21.207"
pre-commit = "^3.2.0"
pytest-benchmark = "^4.0.0"

[tool.poetry.group.licenses]
optional = true
Expand Down Expand Up @@ -136,4 +137,4 @@ format-code = "poetry_scripts:format_code"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"
priority = "primary"
Empty file added tests/benchmarking/__init__.py
View file
Empty file.
15 changes: 15 additions & 0 deletions tests/benchmarking/qasm/10qb_ghz.qasm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[10];
creg c[10];
h q[0];
cx q[0], q[1];
cx q[1], q[2];
cx q[2], q[3];
cx q[3], q[4];
cx q[4], q[5];
cx q[5], q[6];
cx q[6], q[7];
cx q[7], q[8];
cx q[8], q[9];
measure q -> c;
160 changes: 160 additions & 0 deletions tests/benchmarking/qasm/10qb_random_cnot.qasm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[10];
creg c[10];
u3(2.1476154995782335, 3.8199553784163194, 4.2454916175770645) q[0];
u3(4.050215034506989, 4.760568095907491, 3.396091016455767) q[1];
u3(4.386798875432581, 2.64391401939085, 4.641209421796389) q[2];
u3(6.188669264630735, 1.5232788292469255, 3.5987642076340554) q[3];
u3(5.091820489910071, 4.55352107708396, 5.3092933432957485) q[4];
u3(5.2501596312696055, 1.115614436416206, 1.2773565665236108) q[5];
u3(0.8159055856276346, 3.112855881632552, 0.045396816840497234) q[6];
u3(5.424173744031299, 3.9670950484042677, 3.6761161503095257) q[7];
u3(0.5465152234834442, 5.910844912794045, 0.5812240803011206) q[8];
u3(2.0187305348565685, 6.16030364236002, 2.4360975620535594) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(0.8123125007312298, 0.6148585070167231, 0.736178782135525) q[0];
u3(0.7659003651527424, 0.37631704599906773, 0.6213598995356535) q[1];
u3(3.0287212326490884, 0.9626311703384103, 4.3720082182605005) q[2];
u3(2.3548897357774825, 1.860572080686291, 4.255155410793541) q[3];
u3(1.071948354056817, 4.385335985468801, 5.0688745154886465) q[4];
u3(0.1032826341991083, 0.38305043965672037, 3.5132076404261845) q[5];
u3(3.6192712801930718, 3.283590670591691, 1.1168060117763603) q[6];
u3(0.5386542989000254, 1.0780695053112679, 5.920418397929598) q[7];
u3(4.616823999533406, 5.643020446897611, 1.7324997416884562) q[8];
u3(4.590248277985277, 4.9316630926502025, 1.1061214017930179) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(6.146355263634162, 1.8999513184139087, 4.140355349266817) q[0];
u3(4.906129195299491, 3.5386500090489315, 2.786243507890957) q[1];
u3(0.6332830638217943, 0.8080387767603451, 5.61453144838349) q[2];
u3(2.0265652535084135, 1.3290438198905894, 2.854055432954448) q[3];
u3(1.214652215013949, 4.651629128387061, 3.6946059819323187) q[4];
u3(2.5204389853308142, 4.262259888043107, 0.9711069088036576) q[5];
u3(4.36953863846858, 5.679293679068043, 1.0594444892973758) q[6];
u3(5.699848450751837, 0.9082555675439579, 2.8830377987325586) q[7];
u3(2.087886534654373, 4.7096739732597666, 5.079764263317569) q[8];
u3(1.1734692712643875, 3.8881137321160946, 0.9838633326741538) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(2.3740805458166356, 2.4029591583281475, 0.9264585610627256) q[0];
u3(6.067304332958309, 0.7616606594247852, 2.4431081228394422) q[1];
u3(2.960847198635117, 4.061271917629743, 4.748873349452179) q[2];
u3(1.7782683551480225, 3.6497020269990212, 1.6071935625390077) q[3];
u3(2.051994990919352, 6.200262081428702, 4.77979310091581) q[4];
u3(2.8951789242569568, 1.3338407285774465, 3.723042367618023) q[5];
u3(2.429195063286592, 1.0847451099950298, 6.279793015219683) q[6];
u3(2.3574921290684263, 0.8618573915600358, 0.7776453195158278) q[7];
u3(1.174901194944264, 3.977845955068551, 0.8392630346355411) q[8];
u3(2.309645003060026, 3.669075644631806, 1.1475858700376442) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(4.208208423347275, 0.44816107770742314, 0.17658686915098662) q[0];
u3(3.4373975068726317, 0.1793550902470097, 0.16944209301730268) q[1];
u3(3.3456844156739964, 3.3529347180608506, 0.8860579033639527) q[2];
u3(0.6891308281239467, 0.3975281392651653, 4.394096046794606) q[3];
u3(5.2908557014895745, 4.852062048472861, 5.973771579580277) q[4];
u3(2.1698269790851628, 5.84268546572853, 1.3697216809734303) q[5];
u3(3.7828064618099684, 1.8454046390083347, 3.0422440207823356) q[6];
u3(0.5191331897614178, 0.1383746265796521, 5.323959052628703) q[7];
u3(6.022428841569019, 5.6878832191083735, 0.5366042001812283) q[8];
u3(2.998161287591667, 2.9341129601556086, 5.365576467493857) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(4.2792532385518465, 0.894384316103267, 4.193079822308461) q[0];
u3(3.7053658702424195, 2.4414672438270344, 6.122309061250553) q[1];
u3(2.060756357985279, 2.384081686664948, 5.757840289977498) q[2];
u3(2.979885458783252, 4.019500862667376, 5.657160320507938) q[3];
u3(0.9250294211968126, 3.496146387987904, 2.6521585347834273) q[4];
u3(6.034403602281088, 5.058571186352238, 2.7662121247866693) q[5];
u3(5.636283759966305, 1.4370592490921679, 0.315001508528092) q[6];
u3(2.9966852179494703, 5.586758879161887, 2.1719942215981107) q[7];
u3(2.417281132904969, 2.920986433111673, 5.442531735681578) q[8];
u3(5.41136419070043, 6.007338996139764, 1.5049790947891035) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(1.7023252396545367, 1.9478277883510777, 3.7612624863801885) q[0];
u3(1.3791631757063214, 3.555088714605354, 2.1381067388331894) q[1];
u3(1.0549623721801304, 2.3081802603833235, 1.7649096051373347) q[2];
u3(3.08985673105197, 1.3611914784707968, 5.058618478105476) q[3];
u3(1.186425672751201, 4.947775678749102, 1.7106267514699436) q[4];
u3(0.2777347545847968, 1.7377284408840898, 2.6649728231716017) q[5];
u3(3.3694436899699918, 4.879476043461998, 3.8767038389527166) q[6];
u3(6.222063562880843, 2.8587386021573886, 0.41863481802587604) q[7];
u3(1.2705682887507463, 1.378397408994558, 2.684218142207635) q[8];
u3(2.6729994642440182, 4.557845767622726, 4.307520241147747) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(4.100123836325104, 5.022873542927932, 3.2667802215994484) q[0];
u3(0.028587224951316596, 2.070721842216032, 5.441036586555664) q[1];
u3(2.367014332211276, 2.0076359530127075, 2.961320395665276) q[2];
u3(2.8098289719214384, 5.676971378883951, 2.125915022677554) q[3];
u3(1.4220484049078315, 2.1748026183219107, 2.0444745734014247) q[4];
u3(2.0847493568038016, 0.6880625313901045, 3.7645129853317725) q[5];
u3(5.752479239420571, 1.321165864641857, 0.8045918772660725) q[6];
u3(2.8876429865552002, 4.228553816959603, 4.380084516121471) q[7];
u3(5.970710235368468, 0.33358964932408525, 5.997145881049767) q[8];
u3(5.858111162898702, 1.1881820771053189, 5.2159468150357275) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(2.0128820703743537, 2.5490107473433534, 0.8636461222743614) q[0];
u3(0.21518649972015771, 2.0768366561032545, 0.7383645523431643) q[1];
u3(5.506620884435293, 6.008115491486742, 2.3630874422671955) q[2];
u3(4.2799588832852535, 2.228753936083146, 2.3984018539348693) q[3];
u3(4.703895803505276, 1.2432363321460245, 3.256798195471329) q[4];
u3(2.0516250087859556, 4.063325153592703, 1.5659145082601382) q[5];
u3(5.731311489439932, 3.971757328214825, 4.217152990704095) q[6];
u3(5.776907807401078, 0.9526679512736421, 2.14271080135232) q[7];
u3(3.938897024600601, 5.92895250274184, 1.0905368642061466) q[8];
u3(2.1985111660832235, 2.0839506701568915, 5.540189268629577) q[9];
cx q[1], q[2];
cx q[3], q[4];
cx q[5], q[6];
cx q[7], q[8];
u3(0.12796584980929313, 5.826913214957174, 2.7155071351451276) q[0];
u3(5.756491129158667, 4.8030432559759975, 4.8576917393240695) q[1];
u3(0.7649831042720576, 5.47203513006738, 0.7096207234473794) q[2];
u3(3.7899238968036735, 5.852289804611468, 0.2777690187744513) q[3];
u3(2.5041425413541387, 1.5770973295757982, 6.125344218237444) q[4];
u3(3.705870575034715, 6.040125500880039, 2.760153129150555) q[5];
u3(0.7441818812294555, 4.231228485233901, 6.2813647792398575) q[6];
u3(5.221783683598857, 2.2955817305033346, 1.4787114225665745) q[7];
u3(3.237672276846762, 0.6536479042046223, 0.12725564980679163) q[8];
u3(2.5464418661234167, 1.087589107809967, 3.87116512335739) q[9];
cx q[0], q[1];
cx q[2], q[3];
cx q[4], q[5];
cx q[6], q[7];
cx q[8], q[9];
u3(4.506939621598436, 0.5274988381397553, 6.237201066255194) q[0];
u3(0.9916102148500096, 0.5632238245218942, 3.6191342928509647) q[1];
u3(3.3029262476713885, 5.271532983257014, 2.9899958917558664) q[2];
u3(0.10108438587549094, 1.0216049899008273, 5.6769838692552135) q[3];
u3(3.807222096043554, 3.7243918019820113, 0.8943134830623742) q[4];
u3(3.509084223409552, 1.4239459680268831, 4.42268381675587) q[5];
u3(0.5854954719853341, 1.8346266000931504, 3.5712194229497247) q[6];
u3(0.8259591927508211, 0.7904851976172592, 2.7642246833611175) q[7];
u3(5.692561103875125, 5.2303069022442905, 1.5362328399346596) q[8];
u3(0.9448399469329687, 5.089689938085127, 4.762931686091827) q[9];
measure q -> c;
69 changes: 69 additions & 0 deletions tests/benchmarking/qasm/2qb_clifford.qasm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
OPENQASM 2.0;
include "qelib1.inc";
gate gate_Clifford_2Q_5451_ q0,q1 { h q1; h q0; s q0; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_5896_ q0,q1 { h q0; cx q0,q1; h q0; s q0; sdg q1; h q1; y q0; }
gate gate_Clifford_2Q_8699_ q0,q1 { h q0; h q1; cx q0,q1; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_10949_ q0,q1 { h q0; h q1; h q0; s q0; sdg q1; h q1; cx q0,q1; sdg q0; h q0; x q0; x q1; }
gate gate_Clifford_2Q_401_ q0,q1 { sdg q1; h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; x q1; }
gate gate_Clifford_2Q_1660_ q0,q1 { sdg q0; h q0; h q1; s q1; cx q0,q1; h q1; s q1; z q0; }
gate gate_Clifford_2Q_9480_ q0,q1 { h q0; h q1; h q1; s q1; cx q0,q1; cx q1,q0; h q1; s q1; y q0; }
gate gate_Clifford_2Q_10928_ q0,q1 { h q0; h q1; h q0; s q0; sdg q1; h q1; cx q0,q1; h q1; s q1; }
gate gate_Clifford_2Q_2871_ q0,q1 { h q0; s q0; h q1; s q1; cx q0,q1; cx q1,q0; cx q0,q1; x q0; z q1; }
gate gate_Clifford_2Q_3592_ q0,q1 { h q1; h q1; s q1; cx q0,q1; sdg q0; h q0; y q0; }
gate gate_Clifford_2Q_10011_ q0,q1 { h q0; h q1; sdg q0; h q0; sdg q1; h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; y q0; z q1; }
gate gate_Clifford_2Q_4876_ q0,q1 { h q1; h q0; s q0; cx q0,q1; sdg q0; h q0; z q0; }
gate gate_Clifford_2Q_3146_ q0,q1 { h q1; cx q0,q1; cx q1,q0; h q0; s q0; y q0; y q1; }
gate gate_Clifford_2Q_9535_ q0,q1 { h q0; h q1; h q1; s q1; cx q0,q1; cx q1,q0; sdg q0; h q0; h q1; s q1; z q0; z q1; }
gate gate_Clifford_2Q_2960_ q0,q1 { h q1; cx q0,q1; sdg q0; h q0; sdg q1; h q1; }
gate gate_Clifford_2Q_4713_ q0,q1 { h q1; sdg q0; h q0; h q1; s q1; cx q0,q1; cx q1,q0; sdg q0; h q0; sdg q1; h q1; y q0; x q1; }
gate gate_Clifford_2Q_7416_ q0,q1 { h q0; sdg q0; h q0; h q1; s q1; cx q0,q1; h q1; s q1; y q0; }
gate gate_Clifford_2Q_6331_ q0,q1 { h q0; sdg q1; h q1; cx q0,q1; cx q1,q0; sdg q0; h q0; h q1; s q1; y q0; z q1; }
gate gate_Clifford_2Q_987_ q0,q1 { sdg q0; h q0; cx q0,q1; y q0; z q1; }
gate gate_Clifford_2Q_317_ q0,q1 { cx q0,q1; cx q1,q0; cx q0,q1; z q0; x q1; }
gate gate_Clifford_2Q_976_ q0,q1 { sdg q0; h q0; cx q0,q1; }
qreg q[2];
creg meas[2];
gate_Clifford_2Q_5451_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_5896_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_8699_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10949_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_401_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_1660_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_9480_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10928_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_2871_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_3592_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_10011_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_4876_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_3146_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_9535_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_2960_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_4713_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_7416_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_6331_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_987_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_317_ q[0],q[1];
barrier q[0],q[1];
gate_Clifford_2Q_976_ q[0],q[1];
barrier q[0],q[1];
measure q[0] -> meas[0];
measure q[1] -> meas[1];
37 changes: 37 additions & 0 deletions tests/benchmarking/qasm/2qb_random_cnot.qasm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[2];
creg c[2];
u3(1.3006602139645063, 0.534181202485765, 5.689257839837927) q[0];
u3(0.6435923222995911, 2.6357056328036563, 4.583407399749809) q[1];
cx q[0], q[1];
u3(0.5469197324369987, 5.838208081346666, 4.8559930708683225) q[0];
u3(2.1773336422972918, 1.463075857751379, 1.7386307630262692) q[1];
cx q[0], q[1];
u3(0.08906857576487162, 2.0524090814689617, 3.924823508136547) q[0];
u3(5.691523965972233, 4.745449389658818, 3.618032619773981) q[1];
cx q[0], q[1];
u3(1.6052519941473888, 1.7146648534578364, 2.7581683498790937) q[0];
u3(3.695617942720016, 1.4719988957904238, 4.722941564499853) q[1];
cx q[0], q[1];
u3(5.593711172292128, 4.14687486034386, 3.047863806461297) q[0];
u3(2.3259486973738075, 1.8055595481953204, 2.830280329583103) q[1];
cx q[0], q[1];
u3(3.0508523408675328, 5.782789478403682, 5.782316802038208) q[0];
u3(5.309221711611476, 3.59325273719956, 0.7510463983108712) q[1];
cx q[0], q[1];
u3(2.577591983152861, 4.873374951287487, 1.1955443415738125) q[0];
u3(2.7029681627604765, 0.5410471721128901, 2.164425117559505) q[1];
cx q[0], q[1];
u3(1.580605669989407, 1.0969371557584393, 4.2352542755153175) q[0];
u3(5.29713698252154, 0.8143890512769186, 4.535540282844311) q[1];
cx q[0], q[1];
u3(2.98211362929654, 0.6985239567305469, 4.271335180649374) q[0];
u3(0.3828013976141185, 6.142020345273884, 2.9774674667682075) q[1];
cx q[0], q[1];
u3(3.286579382426756, 3.3449235001761486, 2.1157006056982524) q[0];
u3(3.6113600392886864, 4.34759595156976, 2.9588514547434657) q[1];
cx q[0], q[1];
u3(0.4285650460830407, 1.5673997888781974, 5.446012275807939) q[0];
u3(5.312063183960821, 4.807739604050404, 5.4774519419546515) q[1];
measure q -> c;
7 changes: 7 additions & 0 deletions tests/benchmarking/qasm/bell_state.qasm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
OPENQASM 2.0;
include "qelib1.inc";
qreg q[2];
creg c[2];
h q[0];
cx q[0], q[1];
measure q -> c;
Loading

0 comments on commit de1ab83

Please sign in to comment.