From 5470da1b7b54e464fefe3dc40bb74c4da19669c6 Mon Sep 17 00:00:00 2001 From: qxzhou1010 Date: Mon, 12 Aug 2024 16:25:53 +0800 Subject: [PATCH 1/3] Add PSI v2 benchmark --- README.md | 6 + docs/user_guide/psi_v2_benchmark.md | 421 ++++++++++++++++++++++++++++ 2 files changed, 427 insertions(+) create mode 100644 docs/user_guide/psi_v2_benchmark.md diff --git a/README.md b/README.md index cb956b4..7ba101c 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,9 @@ chmod +x traceconv ./traceconv json [trace file path] [json file path] ``` 4. Open chrome://tracing in your chrome and load JSON file. + + + +## PSI V2 Benchamrk + +Please refer to [PSI V2 Benchmark](docs/user_guide/psi_v2_benchmark.md) \ No newline at end of file diff --git a/docs/user_guide/psi_v2_benchmark.md b/docs/user_guide/psi_v2_benchmark.md new file mode 100644 index 0000000..0c799af --- /dev/null +++ b/docs/user_guide/psi_v2_benchmark.md @@ -0,0 +1,421 @@ +# PSI V2 Benchmark + +This document will introduce the PSI V2 Benchmark. It uses the PSI V2 version of the interface. +## Building from source + +``` +git clone https://github.com/secretflow/psi.git +cd psi +bazel build //... -c opt +``` + +If the building is successful, you will find an executable file named `main` in the 'bazel-bin/psi' directory. We will use `./main` file, combined with config file to run different PSI protocols. Such as: + +``` +./main --config configs/sender.config +./main --config configs/receiver.config + +``` +## Generate Data + + +In order to measure the performance of different PSI protocols under different data scales, we need to generate dummy data through [generate_psi.py](https://github.com/secretflow/secretflow/blob/main/docs/developer/benchmark/resources/generate_psi.py) + +```python +python3 generate_psi.py 10000000 10000000 + +python3 generate_psi.py 100000000 100000000 +``` + +## Prepare config file + +We use the config file to specify different PSI protocols and input data. + +### sender.config +``` +{ + "psi_config": { + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_SENDER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_1_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/sender/sender_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/sender/sender.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } + }, + "link_config": { + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "sender" + } +``` + + +### receiver.config +``` +{ + "psi_config": { + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_RECEIVER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_2_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/receiver/receiver_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/receiver/receiver.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } + }, + "link_config": { + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "receiver" + } +``` + + +## Run PSI with docker + +In order to measure the PSI V2 benchmark under different machine configurations and network configurations, we use two dockers to act as sender and receiver respectively. + + +**alice** + +``` +docker run -d -it --name "alice" \ + --mount type=bind,source="$(pwd)",target=/home/admin/dev/ \ + -w /home/admin/dev \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --cap-add=NET_ADMIN \ + --cpuset-cpus 0-31 \ + --privileged=true \ + --memory="64g" \ + --entrypoint="bash" \ + secretflow/ubuntu-base-ci:latest + +docker start alice +docker exec -it alice bash +``` + + +**bob** + +``` +docker run -d -it --name "bob" \ + --mount type=bind,source="$(pwd)",target=/home/admin/dev/ \ + -w /home/admin/dev \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --cap-add=NET_ADMIN \ + --cpuset-cpus 32-63 \ + --privileged=true \ + --memory="64g" \ + --entrypoint="bash" \ + secretflow/ubuntu-base-ci:latest + +docker start bob +docker exec -it bob bash +``` + +## Limit bandwidth and latency + +``` +BAND_WIDTH=100 +burst=128 +DELAY=10 +netem_limit=8000 + +tc qdisc add dev eth0 root handle 1: tbf rate ${BAND_WIDTH}mbit burst ${burst}kb latency 800ms +tc qdisc add dev eth0 parent 1:1 handle 10: netem delay ${DELAY}msec limit ${netem_limit} +``` + + +## Benchmark + +Here we show the PSI V2 Benchmark measured as above. +> The default time unit is seconds, `m` represents minutes, and `h` represents hours. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Machine ConfigurationAlgorithm parametersProtocolNetwork Configuration10 million~10 million100 million to 100 million1 billion to 1 billion
32C64Greceiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_FOURQ',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（FourQ）		LAN424591.45 h
100Mbps/10ms635591.72 h
receiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_25519',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（CURVE_25519）		LAN676972.09 h
100Mbps/10ms696692.08 h
receiver='alice',
protocol='KKRT_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		KKRT_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN383571.13 h
100Mbps/10ms12512583.53 h
receiver='alice',
protocol='RR22_FAST_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_FAST_PSI
_2PC
（BUCKET SIZE IS ONE MILLION）		LAN172250.62 h
100Mbps/10ms444521.43 h
receiver='alice',
protocol='RR22_LOWCOMM_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_LOWCOMM
_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN212230.75 h
100Mbps/10ms383881.21 h
16C32Greceiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_FOURQ',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（FourQ）		LAN464971.66 h
100Mbps/10ms625742.00 h
receiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_25519',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（CURVE_25519）		LAN697012.23 h
100Mbps/10ms747442.49 h
receiver='alice',
protocol='KKRT_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		KKRT_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN373521.18 h
100Mbps/10ms12412283.69 h
receiver='alice',
protocol='RR22_FAST_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_FAST_PSI
_2PC
（BUCKET SIZE IS ONE MILLION）		LAN171780.72 h
100Mbps/10ms434521.70 h
receiver='alice',
protocol='RR22_LOWCOMM_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_LOWCOMM
_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN232160.92 h
100Mbps/10ms403841.52 h
8C16Greceiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_FOURQ',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（FourQ）		LAN666692.6 h
100Mbps/10ms666662.6 h
receiver='alice',
protocol='ECDH_PSI_2PC',
curve_type='CURVE_25519',
precheck_input=False,
sort=False,
broadcast_result=False,		ECDH-PSI-2PC
（CURVE_25519）		LAN11412253.8 h
100Mbps/10ms11211243.8 h
receiver='alice',
protocol='KKRT_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		KKRT_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN373511.6 h
100Mbps/10ms12412114.0 h
receiver='alice',
protocol='RR22_FAST_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_FAST_PSI
_2PC
（BUCKET SIZE IS ONE MILLION）		LAN191661.08 h
100Mbps/10ms444401.91 h
receiver='alice',
protocol='RR22_LOWCOMM_PSI_2PC',
precheck_input=False,
sort=False,
broadcast_result=False,		RR22_LOWCOMM
_PSI_2PC
（BUCKET SIZE IS ONE MILLION）		LAN222041.17 h
100Mbps/10ms373741.72 h
From e0174993eb64b1b244aeb53dcd7a4019dcc6e04f Mon Sep 17 00:00:00 2001 From: qxzhou1010 Date: Mon, 12 Aug 2024 19:04:42 +0800 Subject: [PATCH 2/3] Format config code --- docs/user_guide/psi_v2_benchmark.md | 172 ++++++++++++++-------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/docs/user_guide/psi_v2_benchmark.md b/docs/user_guide/psi_v2_benchmark.md index 0c799af..7ef4016 100644 --- a/docs/user_guide/psi_v2_benchmark.md +++ b/docs/user_guide/psi_v2_benchmark.md @@ -34,98 +34,98 @@ We use the config file to specify different PSI protocols and input data. ### sender.config ``` { - "psi_config": { - "protocol_config": { - "protocol": "PROTOCOL_KKRT", - "role": "ROLE_SENDER", - "broadcast_result": false - }, - "input_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/datas/psi_1_1kw.csv" - }, - "output_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/sender/sender_output.csv" - }, - "keys": [ - "id", - ], - "debug_options": { - "trace_path": "/home/zuoxian/v2psi/sender/sender.trace" - }, - "skip_duplicates_check": false, - "disable_alignment": false, - "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", - "left_side": "ROLE_RECEIVER", - "check_hash_digest": false, - "recovery_config": { - "enabled": false - } - }, - "link_config": { - "parties": [ - { - "id": "receiver", - "host": "127.0.0.1:5300" - }, - { - "id": "sender", - "host": "127.0.0.1:5400" - } - ] - }, - "self_link_party": "sender" - } + "psi_config": { + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_SENDER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_1_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/sender/sender_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/sender/sender.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } + }, + "link_config": { + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "sender" +} ``` ### receiver.config ``` { - "psi_config": { - "protocol_config": { - "protocol": "PROTOCOL_KKRT", - "role": "ROLE_RECEIVER", - "broadcast_result": false - }, - "input_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/datas/psi_2_1kw.csv" - }, - "output_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/receiver/receiver_output.csv" - }, - "keys": [ - "id", - ], - "debug_options": { - "trace_path": "/home/zuoxian/v2psi/receiver/receiver.trace" - }, - "skip_duplicates_check": false, - "disable_alignment": false, - "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", - "left_side": "ROLE_RECEIVER", - "check_hash_digest": false, - "recovery_config": { - "enabled": false - } - }, - "link_config": { - "parties": [ - { - "id": "receiver", - "host": "127.0.0.1:5300" - }, - { - "id": "sender", - "host": "127.0.0.1:5400" - } - ] - }, - "self_link_party": "receiver" - } + "psi_config": { + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_RECEIVER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_2_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/receiver/receiver_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/receiver/receiver.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } + }, + "link_config": { + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "receiver" +} ``` From ba5d1361bb5ccbdf4a3659772532377cea7c3963 Mon Sep 17 00:00:00 2001 From: qxzhou1010 Date: Tue, 13 Aug 2024 09:39:43 +0800 Subject: [PATCH 3/3] Re-format config code --- docs/user_guide/psi_v2_benchmark.md | 148 ++++++++++++++-------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/docs/user_guide/psi_v2_benchmark.md b/docs/user_guide/psi_v2_benchmark.md index 7ef4016..84606c7 100644 --- a/docs/user_guide/psi_v2_benchmark.md +++ b/docs/user_guide/psi_v2_benchmark.md @@ -35,45 +35,45 @@ We use the config file to specify different PSI protocols and input data. ``` { "psi_config": { - "protocol_config": { - "protocol": "PROTOCOL_KKRT", - "role": "ROLE_SENDER", - "broadcast_result": false - }, - "input_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/datas/psi_1_1kw.csv" - }, - "output_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/sender/sender_output.csv" - }, - "keys": [ - "id", - ], - "debug_options": { - "trace_path": "/home/zuoxian/v2psi/sender/sender.trace" - }, - "skip_duplicates_check": false, - "disable_alignment": false, - "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", - "left_side": "ROLE_RECEIVER", - "check_hash_digest": false, - "recovery_config": { - "enabled": false - } + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_SENDER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_1_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/sender/sender_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/sender/sender.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } }, "link_config": { - "parties": [ - { - "id": "receiver", - "host": "127.0.0.1:5300" - }, - { - "id": "sender", - "host": "127.0.0.1:5400" - } - ] + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] }, "self_link_party": "sender" } @@ -84,45 +84,45 @@ We use the config file to specify different PSI protocols and input data. ``` { "psi_config": { - "protocol_config": { - "protocol": "PROTOCOL_KKRT", - "role": "ROLE_RECEIVER", - "broadcast_result": false - }, - "input_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/datas/psi_2_1kw.csv" - }, - "output_config": { - "type": "IO_TYPE_FILE_CSV", - "path": "/home/zuoxian/v2psi/receiver/receiver_output.csv" - }, - "keys": [ - "id", - ], - "debug_options": { - "trace_path": "/home/zuoxian/v2psi/receiver/receiver.trace" - }, - "skip_duplicates_check": false, - "disable_alignment": false, - "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", - "left_side": "ROLE_RECEIVER", - "check_hash_digest": false, - "recovery_config": { - "enabled": false - } + "protocol_config": { + "protocol": "PROTOCOL_KKRT", + "role": "ROLE_RECEIVER", + "broadcast_result": false + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/datas/psi_2_1kw.csv" + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/home/zuoxian/v2psi/receiver/receiver_output.csv" + }, + "keys": [ + "id", + ], + "debug_options": { + "trace_path": "/home/zuoxian/v2psi/receiver/receiver.trace" + }, + "skip_duplicates_check": false, + "disable_alignment": false, + "advanced_join_type": "ADVANCED_JOIN_TYPE_UNSPECIFIED", + "left_side": "ROLE_RECEIVER", + "check_hash_digest": false, + "recovery_config": { + "enabled": false + } }, "link_config": { - "parties": [ - { - "id": "receiver", - "host": "127.0.0.1:5300" - }, - { - "id": "sender", - "host": "127.0.0.1:5400" - } - ] + "parties": [ + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] }, "self_link_party": "receiver" }