diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5c207fa5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +obj +ext + +deploy-run.conf + +bin +src \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..d06ff524 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,106 @@ +{ + "files.associations": { + "csignal": "cpp", + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "array": "cpp", + "atomic": "cpp", + "hash_map": "cpp", + "hash_set": "cpp", + "strstream": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "chrono": "cpp", + "codecvt": "cpp", + "compare": "cpp", + "complex": "cpp", + "concepts": "cpp", + "condition_variable": "cpp", + "cstdint": "cpp", + "deque": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "string": "cpp", + "unordered_map": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "numbers": "cpp", + "ostream": "cpp", + "semaphore": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "stop_token": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cfenv": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "valarray": "cpp", + "variant": "cpp", + "cassert": "cpp", + "cerrno": "cpp", + "shared_mutex": "cpp", + "__nullptr": "cpp", + "__node_handle": "cpp", + "unordered_set": "cpp", + "span": "cpp", + "regex": "cpp", + "__bit_reference": "cpp", + "__hash_table": "cpp", + "__split_buffer": "cpp", + "__tree": "cpp", + "filesystem": "cpp", + "any": "cpp", + "ios": "cpp", + "charconv": "cpp", + "__bits": "cpp", + "__config": "cpp", + "__debug": "cpp", + "__threading_support": "cpp", + "__tuple": "cpp", + "__verbose_abort": "cpp", + "locale": "cpp", + "queue": "cpp", + "stack": "cpp", + "__functional_base": "cpp", + "__functional_base_03": "cpp", + "__mutex_base": "cpp", + "__string": "cpp" + } +} \ No newline at end of file diff --git a/README b/README index 500adfa6..a248d2ac 100755 --- a/README +++ b/README @@ -70,7 +70,7 @@ Installation And there are some import parameters you need to edit : - src/deployment/main.cc, #define HOT ***: Set amount of Hot records for micorbenchmark, it is used to vary contention index (100 means contention index = 0.01); - - src/sequencer/sequencer.h: #define MAX_BATCH_SIZE *** : Set batch size per 10 ms epoch , set it a little bigger than the actually throughput(200 means every second the sequencer creates 20K transactions) + - src/sequencer/sequencer.h: #define MAX_LOCK_BATCH_SIZE *** : Set batch size per 10 ms epoch , set it a little bigger than the actually throughput(200 means every second the sequencer creates 20K transactions) You should make sure that your LD_LIBRARY_PATH includes the object files noted in the dependencies above. And you need to edit deploy-run.conf to include the machines which Calvin run on(The port should be same with the port in the src/deployment/portfile). diff --git a/bin/deployment/cluster b/bin/deployment/cluster new file mode 100755 index 00000000..981d90c9 Binary files /dev/null and b/bin/deployment/cluster differ diff --git a/calvin_comparison.sh b/calvin_comparison.sh new file mode 100755 index 00000000..95f81b55 --- /dev/null +++ b/calvin_comparison.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# ============================================================================== +# Calvin 比較実験用スクリプト (96コアサーバー向け) +# ============================================================================== + +# --- 比較対象を設定 ('original' または 'proposed') --- +TARGET="proposed" # ここを 'original' に変えて元カルバンを測定 + +# --- 引数処理 --- +if [ "$1" == "m" ]; then + ARGUMENT="m" +elif [ "$1" == "t" ]; then + ARGUMENT="t" +elif [ "$1" == "y" ]; then + ARGUMENT="y" +else + echo "エラー: 引数が無効です。'm', 't', 'y' のいずれかを使用してください。" + exit 1 +fi +echo "🧪 使用するベンチマーク: $ARGUMENT" +echo "🎯 測定対象: $TARGET" + +# ========================== 実験パラメータ ========================== +if [ "$TARGET" == "original" ]; then + NUM_BACKGROUND=4 + DEFINITIONS_FILE="definitions_original.hh" + SOURCE_DIR="src_calvin" # 元カルバンのソースディレクトリ + OUTPUT_CSV="throughput_summary_${ARGUMENT}_original.csv" +else + # 提案手法 (Dispatcherは1つで実験する例) + NUM_DISPATCHERS=0 + OTHER_BACKGROUND_THREADS=4 + NUM_BACKGROUND=$((OTHER_BACKGROUND_THREADS + NUM_DISPATCHERS)) + DEFINITIONS_FILE="definitions_original.hh" + SOURCE_DIR="src_calvin_ext" # 提案手法のソースディレクトリ + OUTPUT_CSV="throughput_summary_${ARGUMENT}_proposed_d${NUM_DISPATCHERS}.csv" +fi + +# ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ +# ★★★ ここを修正:96コアサーバー向けにテスト範囲を拡張 ★★★ +# ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ +THREAD_COUNTS=(1 4 8 16) + +# <--- 変更点 No.1: 測定回数を60回に設定 --- +RUN_DURATION=10 +NUM_RUNS=4 +WARMUP_RUNS=0 + +# --- 基本設定 --- +SCRIPT_DIR=$(dirname "$0") +cd "$SCRIPT_DIR" || exit 1 +CONFIG_FILE="cygnus-run.conf" +BACKUP_CONFIG_FILE="${CONFIG_FILE}.bak" +LOG_FILE="/tmp/calvin_output.log" + +# --- 事前準備 --- +rm -f /tmp/failed_log_*.log +cp "$CONFIG_FILE" "$BACKUP_CONFIG_FILE" +echo "✅ 設定ファイルをバックアップ: $BACKUP_CONFIG_FILE" +echo "Threads,Average_Throughput(ops/sec)" > "$OUTPUT_CSV" + +# ========================== メインループ ========================== +for THREADS in "${THREAD_COUNTS[@]}"; do + echo -e "\n======== Total Cores for Workers+Dispatchers: $THREADS ========" + + # -------- [ここから修正] -------- + # THREADS を NUM_WORKERS に設定 + NUM_WORKERS=$THREADS + # NUM_CORE を計算 (ワーカー + バックグラウンド) + NUM_CORE=$(($NUM_WORKERS + $NUM_BACKGROUND)) + # -------- [ここまで修正] -------- + + echo " - Configuration: Workers=$NUM_WORKERS, Background=$NUM_BACKGROUND, TotalCores=$NUM_CORE" + + # --- ソースコードと定義ファイルの更新 --- + echo " [1/4] ソースコードと定義ファイルを更新中..." + rm -rf src obj + cp -r "$SOURCE_DIR"/ src + cp "$DEFINITIONS_FILE" src/common/definitions.hh + + # sedで各種マクロを置換 + sed -i -E "s/^#define[[:space:]]+NUM_CORE[[:space:]]+.*$/#define NUM_CORE $NUM_CORE/" src/common/definitions.hh + sed -i -E "s/^#define[[:space:]]+NUM_BACKGROUND_THREADS[[:space:]]+.*$/#define NUM_BACKGROUND_THREADS $NUM_BACKGROUND/" src/common/definitions.hh + sed -i -E "s/^#define[[:space:]]+NUM_WORKERS[[:space:]]+.*$/#define NUM_WORKERS $NUM_WORKERS/" src/common/definitions.hh + if [ "$TARGET" != "original" ]; then + sed -i -E "s/^#define[[:space:]]+NUM_RO_DISPATCHERS[[:space:]]+.*$/#define NUM_RO_DISPATCHERS $NUM_DISPATCHERS/" src/common/definitions.hh + fi + # --- ビルド --- + echo " [2/4] ビルドを実行中..." + cd src + make clean > /dev/null 2>&1 + if make -j$(nproc); then + echo " - ビルド成功。" + else + echo " - ❌ ビルド失敗 (Workers=$NUM_WORKERS)。スキップします。" + cd .. + continue + fi + cd .. + + # ========================== 測定実行 ========================== + echo " [3/4] 測定を実行中 ($NUM_RUNS 回)..." + THROUGHPUT_ARRAY=() + + for i in $(seq 1 $NUM_RUNS); do + ./bin/deployment/db 0 "$ARGUMENT" 0 > "$LOG_FILE" 2>&1 & + APP_PID=$! + sleep "$RUN_DURATION" + kill -SIGINT "$APP_PID" 2>/dev/null + wait "$APP_PID" 2>/dev/null + sleep 1 + if ps -p $APP_PID > /dev/null; then + kill -SIGKILL "$APP_PID" 2>/dev/null + fi + + THROUGHPUT=$(grep -oP 'Completed\s*\K[0-9.]+' "$LOG_FILE" | tail -n 5 | head -n 1 | awk '{print $1}') + + # <--- 変更点 No.2: 最初の10回は結果を保存しない --- + if [ "$i" -le "$WARMUP_RUNS" ]; then + echo " - ウォームアップ実行 $i/$NUM_RUNS: 🚀 完了 (結果は破棄)" + elif [ -z "$THROUGHPUT" ]; then + echo " - 測定実行 $i/$NUM_RUNS: ⚠️ スループット抽出失敗" + FAILED_LOG_NAME="/tmp/failed_log_threads_${THREADS}_run_${i}.log" + mv "$LOG_FILE" "$FAILED_LOG_NAME" + echo " (ログを ${FAILED_LOG_NAME} に保存しました)" + else + echo " - 測定実行 $i/$NUM_RUNS: ✅ スループット: $THROUGHPUT ops/sec" + THROUGHPUT_ARRAY+=($THROUGHPUT) + fi + rm -f "$LOG_FILE" + done + + # ========================== 結果集計 ========================== + echo " [4/4] 結果を集計中..." + # <--- 変更点 No.3: 集計ロジックを修正 --- + NUM_VALID_RUNS=${#THROUGHPUT_ARRAY[@]} + if [ "$NUM_VALID_RUNS" -gt 0 ]; then + # 配列内のすべての有効なスループットを合計する + TOTAL_THROUGHPUT=0 + for t in "${THROUGHPUT_ARRAY[@]}"; do + TOTAL_THROUGHPUT=$(awk "BEGIN {print $TOTAL_THROUGHPUT + $t}") + done + + AVERAGE_THROUGHPUT=$(awk "BEGIN {printf \"%.2f\", $TOTAL_THROUGHPUT / $NUM_VALID_RUNS}") + echo "✅ 平均スループット ($THREADS threads): $AVERAGE_THROUGHPUT ops/sec ($NUM_VALID_RUNS 回の有効実行から算出)" + else + AVERAGE_THROUGHPUT="N/A" + echo "⚠️ $THREADS threads の有効な実行がなかったため、平均を計算できませんでした。" + fi + echo "$THREADS,$AVERAGE_THROUGHPUT" >> "$OUTPUT_CSV" +done + +# ========================== 後始末 ========================== +mv "$BACKUP_CONFIG_FILE" "$CONFIG_FILE" +echo -e "\n🛠️ 設定ファイルを元に戻しました。" +echo "🎉 全実験完了。結果は $OUTPUT_CSV に保存されました。" \ No newline at end of file diff --git a/src_dependent_remote_index/common/configuration_test_one_node.conf b/cygnus-run.conf old mode 100755 new mode 100644 similarity index 62% rename from src_dependent_remote_index/common/configuration_test_one_node.conf rename to cygnus-run.conf index 6f5083ba..0998649c --- a/src_dependent_remote_index/common/configuration_test_one_node.conf +++ b/cygnus-run.conf @@ -1,2 +1,3 @@ # Node=:::: -node0=0:0:16:128.36.232.50:60001 +node0=0:0:8:133.27.19.50:54564 + diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/configuration_test_one_node.conf b/cygnus-run.conf.bak old mode 100755 new mode 100644 similarity index 62% rename from src_dependent_variable_sized_reconnaissance_phases/common/configuration_test_one_node.conf rename to cygnus-run.conf.bak index 6f5083ba..0998649c --- a/src_dependent_variable_sized_reconnaissance_phases/common/configuration_test_one_node.conf +++ b/cygnus-run.conf.bak @@ -1,2 +1,3 @@ # Node=:::: -node0=0:0:16:128.36.232.50:60001 +node0=0:0:8:133.27.19.50:54564 + diff --git a/definitions_original.hh b/definitions_original.hh new file mode 100644 index 00000000..a461fc6e --- /dev/null +++ b/definitions_original.hh @@ -0,0 +1,101 @@ +#pragma once + +// ============== please don't change ============== +#define COLD_CUTOFF 1000000 +#define MAX_LOCK_BATCH_SIZE 2000 +#define EPOCH_DURATION 0.01 // 0.01 is 10ms +// ================================================= + +// ============== 元のCalvin用設定 (NUMA完全分離版) ============== +// ★ バックグラウンドをNode 1 (奇数)、ワーカーをNode 0 (偶数) に分離 ★ +// ============================================================= + +// ============== server setting ============== +#define NUM_CORE 0 // スクリプトで上書き + +// --- バックグラウンドスレッドの定義 (4つ) --- +#define NUM_BACKGROUND_THREADS 4 +// バックグラウンドをNode 1 (奇数コア) にハードコードで割り当て +#define MULTIPLEXER_CORE 1 +#define SEQUENCER_WRITER_CORE 3 +#define SEQUENCER_READER_CORE 5 +#define LOCK_MANAGER_CORE 7 + +// --- ワーカーの定義 --- +#define NUM_WORKERS_CORE (NUM_CORE - NUM_BACKGROUND_THREADS) +#define NUM_WORKERS (NUM_WORKERS_CORE) + +// -------- [ここから変更] -------- + +/** + * @brief ワーカーコアを割り当てる関数 + * 1. まず、偶数コアを昇順 (0, 2, 4, ...) に割り当てる。 + * 2. 偶数コアを使い切ったら (NUM_COREを超えたら)、 + * 利用可能な奇数コアを降順 (NUM_CORE-1, NUM_CORE-3, ...) に割り当てる。 + * 3. その際、バックグラウンドスレッドが使用するコア (1, 3, 5, 7) は除外する。 + */ +inline int GetWorkerCore(int thread_id) { + // 1. 偶数コアを昇順に割り当て + int even_core = thread_id * 2; + if (even_core < NUM_CORE) { + return even_core; + } + + // 2. 偶数コアを使い切った場合 (スピルオーバー) + // 利用可能な偶数コアの数を計算 + int num_even_cores = (NUM_CORE + 1) / 2; + // 自分は何番目のスピルオーバースレッドか (0-indexed) + int spill_thread_index = thread_id - num_even_cores; + + // 3. 利用可能な奇数コアを降順に探す + // NUM_CORE-1 が偶数なら、(NUM_CORE-1)-1 = NUM_CORE-2 から開始 + int current_odd_core = (NUM_CORE - 1); + if (current_odd_core % 2 == 0) { + current_odd_core--; + } + + int assigned_spill_count = 0; + while (current_odd_core >= 0) { + // バックグラウンドスレッドのコアかチェック + bool is_background_core = + (current_odd_core == MULTIPLEXER_CORE) || + (current_odd_core == SEQUENCER_WRITER_CORE) || + (current_odd_core == SEQUENCER_READER_CORE) || + (current_odd_core == LOCK_MANAGER_CORE); + + if (!is_background_core) { + // 利用可能な奇数コアだった + if (assigned_spill_count == spill_thread_index) { + return current_odd_core; // 該当するコアを返す + } + assigned_spill_count++; + } + current_odd_core -= 2; // 次の奇数コアへ (降順) + } + + // 割り当て失敗 (NUM_WORKERS の定義が正しければ、ここには来ない) + return -1; +} + +// ワーカーをNode 0 (偶数コア) に割り当て +// (ただし、使い切ったら降順の奇数コアにスピルオーバー) +#define GET_WORKER_CORE(thread_id) (GetWorkerCore(thread_id)) +// -------- [ここまで変更] -------- +// ============================================== + +// ============== database setting ============== +#define DB_SIZE 1000000 +#define LOCK_TABLE_SIZE 1000000 +// ============================================== + +// ============== workload setting ============== +#define RW_SET_SIZE 100 +#define SKEW 0.99 +#define UNIFORM_KEY_SELECTION_RATIO 0 +#define HOT 10 +// ============================================== + +// ============== used for only calvin ============== +#define MAX_ACTIVE_TXNS 2000 +#define LOCK_BATCH_SIZE 100 +// ============================================== \ No newline at end of file diff --git a/deploy-run.conf b/deploy-run.conf old mode 100755 new mode 100644 index 2c270bac..eb2d56f1 --- a/deploy-run.conf +++ b/deploy-run.conf @@ -1,10 +1,3 @@ # Node=:::: -node0=0:0:8:128.36.232.18:54564 -node1=0:1:8:128.36.232.12:54564 -node2=0:2:8:128.36.232.15:54564 -node3=0:3:8:128.36.232.20:54564 -node4=0:4:8:128.36.232.19:54564 -node5=0:5:8:128.36.232.34:54564 -node6=0:6:8:128.36.232.14:54564 -node7=0:7:8:128.36.232.16:54564 +node0=0:0:1:133.27.19.50:54564 diff --git a/install-ext b/install-ext index 627f3bc8..4e3ba00e 100755 --- a/install-ext +++ b/install-ext @@ -2,50 +2,63 @@ # requires: autoconf, libtool, libreadline-dev +sudo apt-get update +sudo apt install unzip libtool libtool-bin uuid-dev python-is-python3 python2 +sudo ln -sf /usr/bin/python2 /usr/bin/python # google test ? + +rm -rf ext mkdir ext cd ext #zookeeper -wget http://apache.mirrors.tds.net/zookeeper/current/zookeeper-3.4.6.tar.gz +wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz tar xzf zookeeper-3.4.6.tar.gz rm zookeeper-3.4.6.tar.gz mv zookeeper-3.4.6/src/c zookeeper rm -fr zookeeper-3.4.6 cd zookeeper -./configure --prefix=$PWD +./configure --prefix=$PWD CFLAGS="-Wno-format-overflow -g -O2 -D_GNU_SOURCE" make -j cd .. # protobuf -svn checkout http://protobuf.googlecode.com/svn/trunk/ protobuf +wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz +tar xzf protobuf-2.5.0.tar.gz +rm protobuf-2.5.0.tar.gz +mv protobuf-2.5.0 protobuf cd protobuf -./autogen.sh -./configure --prefix=$PWD +./configure --prefix=$PWD --enable-debug make -j cd .. # googletest -wget http://googletest.googlecode.com/files/gtest-1.6.0.zip +wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/googletest/gtest-1.6.0.zip unzip gtest-1.6.0.zip rm gtest-1.6.0.zip mv gtest-1.6.0 googletest cd googletest -./configure --prefix=$PWD +./configure --prefix=$PWD --enable-debug make -j cd .. # zeromq -wget http://download.zeromq.org/zeromq-2.1.11.tar.gz -tar xzf zeromq-2.1.11.tar.gz -rm zeromq-2.1.11.tar.gz -mv zeromq-2.1.11 zeromq +git clone https://github.com/zeromq/zeromq2-x.git +cd zeromq2-x +git checkout v2.1.11 +cd ../ +mv zeromq2-x zeromq cd zeromq ./autogen.sh -./configure --prefix=$PWD +./configure --prefix=$PWD --enable-debug make -j make install cd .. + + +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/calvin/ext/protobuf/src/.libs +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/calvin/ext/zookeeper/.libs +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/calvin/ext/zeromq/src/.libs diff --git a/src_calvin/Makefile b/src_calvin/Makefile index e3e6e17d..c6b06741 100644 --- a/src_calvin/Makefile +++ b/src_calvin/Makefile @@ -47,7 +47,7 @@ CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ -I$(PROTOB)/src \ -I$(GTEST)/include \ -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror +CXXFLAGS += -Wall -Werror -Wno-format-truncation -Wno-unused-result -Wno-maybe-uninitialized LDFLAGS := -lrt -lpthread $(PG) \ -lprotobuf -L$(PROTOB)/src/.libs \ diff --git a/src_calvin/applications/Makefile.inc b/src_calvin/applications/Makefile.inc index db42fd1b..7bd88aa8 100644 --- a/src_calvin/applications/Makefile.inc +++ b/src_calvin/applications/Makefile.inc @@ -4,7 +4,7 @@ OBJDIRS += applications UPPERC_DIR := APPLICATIONS LOWERC_DIR := applications -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc +APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc applications/ycsb.cc SRC_LINKED_OBJECTS := $(PROTO_OBJS) TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) diff --git a/src_calvin/applications/application.h b/src_calvin/applications/application.h index 1b62b033..f62e1353 100644 --- a/src_calvin/applications/application.h +++ b/src_calvin/applications/application.h @@ -30,7 +30,9 @@ class Application { virtual ~Application() {} // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, + virtual TxnProto* NewTxn(int64 txn_id, + int txn_type, + string args, Configuration* config) const = 0; // Static method to convert a key into an int for an array diff --git a/src_calvin/applications/microbenchmark.cc b/src_calvin/applications/microbenchmark.cc index 93dddedc..4185d1b7 100644 --- a/src_calvin/applications/microbenchmark.cc +++ b/src_calvin/applications/microbenchmark.cc @@ -11,16 +11,20 @@ #include "backend/storage_manager.h" #include "common/utils.h" #include "common/configuration.h" +#include "common/definitions.hh" #include "proto/txn.pb.h" // #define PREFETCHING -#define COLD_CUTOFF 990000 // Fills '*keys' with num_keys unique ints k where // 'key_start' <= k < 'key_limit', and k == part (mod nparts). // Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { +void Microbenchmark::GetRandomKeys(set* keys, + int num_keys, + int key_start, + int key_limit, + int part, + bool is_uniform) { assert(key_start % nparts == 0); keys->clear(); for (int i = 0; i < num_keys; i++) { @@ -28,7 +32,8 @@ void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, int key; do { key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); + nparts * (((is_uniform) ? rand() : zipf_()) % + ((key_limit - key_start) / nparts)); } while (keys->count(key)); keys->insert(key); } @@ -43,7 +48,7 @@ TxnProto* Microbenchmark::InitializeTxn() { txn->set_txn_type(INITIALIZE); // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) + for (int i = 0; i < DB_SIZE; i++) txn->add_write_set(IntToString(i)); return txn; @@ -58,18 +63,17 @@ TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { txn->set_txn_id(txn_id); txn->set_txn_type(MICROTXN_SP); - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); + // // Add one hot key to read/write set. + // int hotkey = part + nparts * (rand() % hot_records); + // txn->add_read_write_set(IntToString(hotkey)); - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. + bool is_uniform = (rand() % 100) < UNIFORM_KEY_SELECTION_RATIO; + + // Insert set of RW_SET_SIZE - 1 random cold keys from specified partition + // into read/write set. set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); + GetRandomKeys(&keys, RW_SET_SIZE, nparts * hot_records, nparts * DB_SIZE, + part, is_uniform); // uniform dist for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); @@ -92,21 +96,17 @@ TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { txn->add_read_write_set(IntToString(hotkey1)); txn->add_read_write_set(IntToString(hotkey2)); - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into + bool is_uniform = (rand() % 100) < UNIFORM_KEY_SELECTION_RATIO; + + // Insert set of RW_SET_SIZE/2 - 1 random cold keys from each partition into // read/write set. set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); + GetRandomKeys(&keys, RW_SET_SIZE / 2 - 1, nparts * hot_records, + nparts * DB_SIZE, part1, is_uniform); for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); + GetRandomKeys(&keys, RW_SET_SIZE / 2 - 1, nparts * hot_records, + nparts * DB_SIZE, part2, is_uniform); for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); @@ -115,8 +115,10 @@ TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { // The load generator can be called externally to return a transaction proto // containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { +TxnProto* Microbenchmark::NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration* config) const { return NULL; } @@ -124,31 +126,29 @@ int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { // Read all elements of 'txn->read_set()', add one to each, write them all // back out. - for (int i = 0; i < kRWSetSize; i++) { + for (int i = 0; i < RW_SET_SIZE; i++) { Value* val = storage->ReadObject(txn->read_write_set(i)); *val = IntToString(StringToInt(*val) + 1); // Not necessary since storage already has a pointer to val. // storage->PutObject(txn->read_write_set(i), val); - // The following code is for microbenchmark "long" transaction, uncomment it if for "long" transaction + // The following code is for microbenchmark "long" transaction, uncomment it + // if for "long" transaction /**int x = 1; for(int i = 0; i < 1100; i++) { x = x*x+1; x = x+10; x = x-2; }**/ - } return 0; } void Microbenchmark::InitializeStorage(Storage* storage, Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { + for (int i = 0; i < nparts * DB_SIZE; i++) { if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { storage->PutObject(IntToString(i), new Value(IntToString(i))); - } } } - diff --git a/src_calvin/applications/microbenchmark.h b/src_calvin/applications/microbenchmark.h index a6dd675a..5c859ab2 100644 --- a/src_calvin/applications/microbenchmark.h +++ b/src_calvin/applications/microbenchmark.h @@ -11,26 +11,35 @@ #include #include "applications/application.h" +#include "common/definitions.hh" +#include "common/random.hh" +#include "common/zipf.hh" using std::set; using std::string; class Microbenchmark : public Application { public: + Xoroshiro128Plus rnd_; + FastZipf zipf_; + enum TxnType { INITIALIZE = 0, MICROTXN_SP = 1, MICROTXN_MP = 2, }; - Microbenchmark(int nodecount, int hotcount) { + Microbenchmark(int nodecount, int hotcount) + : rnd_(), zipf_(&rnd_, SKEW, DB_SIZE) { nparts = nodecount; hot_records = hotcount; } virtual ~Microbenchmark() {} - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, + virtual TxnProto* NewTxn(int64 txn_id, + int txn_type, + string args, Configuration* config = NULL) const; virtual int Execute(TxnProto* txn, StorageManager* storage) const; @@ -40,16 +49,17 @@ class Microbenchmark : public Application { int nparts; int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} + void GetRandomKeys(set* keys, + int num_keys, + int key_start, + int key_limit, + int part, + bool is_uniform); + Microbenchmark() : rnd_(), zipf_(&rnd_, SKEW, DB_SIZE) {} }; #endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_calvin/applications/tpcc.cc b/src_calvin/applications/tpcc.cc index fa29cd36..07bca5b6 100644 --- a/src_calvin/applications/tpcc.cc +++ b/src_calvin/applications/tpcc.cc @@ -19,1099 +19,1180 @@ using std::string; // ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } +unordered_map ItemList; +Value *TPCC::GetItem(Key key) const +{ + return ItemList[key]; +} +void TPCC::SetItem(Key key, Value *value) const +{ + ItemList[key] = value; +} // The load generator can be called externally to return a // transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { +TxnProto *TPCC::NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration *config) const +{ + // Create the new transaction object + TxnProto *txn = new TxnProto(); + + // Set the transaction's standard attributes + txn->set_txn_id(txn_id); + txn->set_txn_type(txn_type); + txn->set_isolation_level(TxnProto::SERIALIZABLE); + txn->set_status(TxnProto::NEW); + txn->set_multipartition(false); + + // Parse out the arguments to the transaction + TPCCArgs *txn_args = new TPCCArgs(); + assert(txn_args->ParseFromString(args)); + bool mp = txn_args->multipartition(); + int remote_node; + if (mp) + { + do + { + remote_node = rand() % config->all_nodes.size(); + } while (config->all_nodes.size() > 1 && + remote_node == config->this_node_id); + } + + // Create an arg list + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->set_system_time(GetTime()); + + // Because a switch is not scoped we declare our variables outside of it + int warehouse_id, district_id, customer_id; + char warehouse_key[128], district_key[128], customer_key[128]; + int order_line_count; + bool invalid; + Value customer_value; + std::set items_used; + + // We set the read and write set based on type + switch (txn_type) + { // Initialize case INITIALIZE: - // Finished with INITIALIZE txn creation - break; + // Finished with INITIALIZE txn creation + break; // New Order case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } + // First, we pick a local warehouse + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); + txn->add_read_set(warehouse_key); - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); + // Next, we pick a random district + district_id = rand() % DISTRICTS_PER_WAREHOUSE; + snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, + district_id); + txn->add_read_write_set(district_key); - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); + // Finally, we pick a random customer + customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", warehouse_id, + district_id, customer_id); + txn->add_read_set(customer_key); - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); + int order_number; + if (next_order_id_for_district.count(district_key) > 0) + { + order_number = next_order_id_for_district[district_key]; + next_order_id_for_district[district_key]++; + } + else + { + order_number = 0; + next_order_id_for_district[district_key] = 1; + } - } + // We set the length of the read and write set uniformly between 5 and 15 + order_line_count = (rand() % 11) + 5; + + // Let's choose a bad transaction 1% of the time + invalid = false; + // if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) + // invalid = true; + + // Iterate through each order line + for (int i = 0; i < order_line_count; i++) + { + // Set the item id (Invalid orders have the last item be -1) + int item; + do + { + item = rand() % NUMBER_OF_ITEMS; + } while (items_used.count(item) > 0); + items_used.insert(item); + + if (invalid && i == order_line_count - 1) + item = -1; + + // Print the item key into a buffer + char item_key[128]; + snprintf(item_key, sizeof(item_key), "i%d", item); + + // Create an order line warehouse key (default is local) + char remote_warehouse_key[128]; + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "%s", + warehouse_key); + + // We only do ~1% remote transactions + if (mp) + { + txn->set_multipartition(true); + + // We loop until we actually get a remote one + int remote_warehouse_id; + do + { + remote_warehouse_id = + rand() % (WAREHOUSES_PER_NODE * config->all_nodes.size()); + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", + remote_warehouse_id); + } while (config->all_nodes.size() > 1 && + config->LookupPartition(remote_warehouse_key) != + remote_node); + } + + // Determine if we should add it to read set to avoid duplicates + bool needed = true; + for (int j = 0; j < txn->read_set_size(); j++) + { + if (txn->read_set(j) == remote_warehouse_key) + needed = false; + } + if (needed) + txn->add_read_set(remote_warehouse_key); + + // Finally, we set the stock key to the read and write set + Key stock_key = string(remote_warehouse_key) + "s" + item_key; + txn->add_read_write_set(stock_key); + + // Set the quantity randomly within [1..10] + tpcc_args->add_quantities(rand() % 10 + 1); + + // Finally, we add the order line key to the write set + char order_line_key[128]; + snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", + district_key, order_number, i); + txn->add_write_set(order_line_key); + } - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); + // Create a new order key to add to write set + char new_order_key[128]; + snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, + order_number); + txn->add_write_set(new_order_key); - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); + // Create an order key to add to write set + char order_key[128]; + snprintf(order_key, sizeof(order_key), "%so%d", district_key, + order_number); + txn->add_write_set(order_key); - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; + // Set the order line count in the args + tpcc_args->add_order_line_count(order_line_count); + tpcc_args->set_order_number(order_number); + break; // Payment case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } + // Specify an amount for the payment + tpcc_args->set_amount( + rand() / (static_cast(RAND_MAX + 1.0)) * 4999.0 + 1); - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); + // First, we pick a local warehouse + + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", warehouse_id); + txn->add_read_write_set(warehouse_key); + + // Next, we pick a district + district_id = rand() % DISTRICTS_PER_WAREHOUSE; + snprintf(district_key, sizeof(district_key), "w%dd%dy", warehouse_id, + district_id); + txn->add_read_write_set(district_key); + + // Add history key to write set + char history_key[128]; + snprintf(history_key, sizeof(history_key), "w%dh%ld", warehouse_id, + txn->txn_id()); + txn->add_write_set(history_key); + + // Next, we find the customer as a local one + if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) + { + customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", warehouse_id, + district_id, customer_id); + + // If the probability is 15%, we make it a remote customer + } + else + { + int remote_warehouse_id; + int remote_district_id; + int remote_customer_id; + char remote_warehouse_key[40]; + do + { + remote_warehouse_id = + rand() % (WAREHOUSES_PER_NODE * config->all_nodes.size()); + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", + remote_warehouse_id); + + remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; + + remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", + remote_warehouse_id, remote_district_id, remote_customer_id); + } while (config->all_nodes.size() > 1 && + config->LookupPartition(remote_warehouse_key) != remote_node); + } + + // We only do secondary keying ~60% of the time + if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) + { + // Now that we have the object, let's create the txn arg + tpcc_args->set_last_name(customer_key); + txn->add_read_set(customer_key); + + // Otherwise just give a customer key + } + else + { + txn->add_read_write_set(customer_key); + } + + break; + + case ORDER_STATUS: + { + string customer_string; + string customer_latest_order; + string warehouse_string; + string district_string; + int customer_order_line_number; + + if (latest_order_id_for_customer.size() < 1) + { + txn->set_txn_id(-1); + break; + } + + pthread_mutex_lock(&mutex_); + customer_string = + (*involed_customers)[rand() % involed_customers->size()]; + pthread_mutex_unlock(&mutex_); + customer_latest_order = latest_order_id_for_customer[customer_string]; + warehouse_string = customer_string.substr(0, customer_string.find("d")); + district_string = customer_string.substr(0, customer_string.find("c")); + snprintf(customer_key, sizeof(customer_key), "%s", + customer_string.c_str()); + snprintf(warehouse_key, sizeof(warehouse_key), "%s", + warehouse_string.c_str()); + snprintf(district_key, sizeof(district_key), "%s", + district_string.c_str()); + + customer_order_line_number = order_line_number[customer_latest_order]; + txn->add_read_set(warehouse_key); + txn->add_read_set(district_key); txn->add_read_set(customer_key); - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; + snprintf(order_key, sizeof(order_key), "%s", + customer_latest_order.c_str()); + txn->add_read_set(order_key); + char order_line_key[128]; + for (int i = 0; i < customer_order_line_number; i++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + i); + txn->add_read_set(order_line_key); + } + + tpcc_args->add_order_line_count(customer_order_line_number); + + break; } + case STOCK_LEVEL: + { + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - // Next, we pick a random district district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } + snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, + district_id); + + if (latest_order_id_for_district.count(district_key) == 0) + { + txn->set_txn_id(-1); + break; + } + + txn->add_read_set(warehouse_key); + txn->add_read_set(district_key); + int latest_order_number = latest_order_id_for_district[district_key]; + char order_line_key[128]; + char stock_key[128]; + + tpcc_args->set_lastest_order_number(latest_order_number); + tpcc_args->set_threshold(rand() % 10 + 10); + + for (int i = latest_order_number; + (i >= 0) && (i > latest_order_number - 20); i--) + { + snprintf(order_key, sizeof(order_key), "%so%d", district_key, i); + int ol_number = order_line_number[order_key]; + + for (int j = 0; j < ol_number; j++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + j); + int item = item_for_order_line[order_line_key]; + if (items_used.count(item) > 0) + { + continue; + } + items_used.insert(item); + txn->add_read_set(order_line_key); + snprintf(stock_key, sizeof(stock_key), "%ssi%d", warehouse_key, item); + txn->add_read_set(stock_key); + } + } + + break; + } + + case DELIVERY: + { + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); + txn->add_read_set(warehouse_key); + + char order_line_key[128]; + int oldest_order; + + for (int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) + { + snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); + if ((smallest_order_id_for_district.count(district_key) == 0) || + (smallest_order_id_for_district[district_key] > + latest_order_id_for_district[district_key])) + { + continue; + } + else + { + txn->add_read_set(district_key); + oldest_order = smallest_order_id_for_district[district_key]; + smallest_order_id_for_district[district_key]++; + + snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, + oldest_order); + txn->add_read_write_set(new_order_key); + } + + snprintf(order_key, sizeof(order_key), "%so%d", district_key, + oldest_order); + txn->add_read_write_set(order_key); + int ol_number = order_line_number[order_key]; + tpcc_args->add_order_line_count(ol_number); + + for (int j = 0; j < ol_number; j++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + j); + txn->add_read_write_set(order_line_key); + } + + snprintf(customer_key, sizeof(customer_key), "%s", + customer_for_order[order_key].c_str()); + txn->add_read_write_set(customer_key); + } + + break; + } // Invalid transaction default: - break; - } + break; + } - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); + // Set the transaction's args field to a serialized version + Value args_string; + assert(tpcc_args->SerializeToString(&args_string)); + txn->set_arg(args_string); - // Free memory - delete tpcc_args; - delete txn_args; + // Free memory + delete tpcc_args; + delete txn_args; - return txn; + return txn; } // The execute function takes a single transaction proto and executes it based // on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { +int TPCC::Execute(TxnProto *txn, StorageManager *storage) const +{ + switch (txn->txn_type()) + { // Initialize case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; + InitializeStorage(storage->GetStorage(), NULL); + return SUCCESS; + break; // New Order case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; + return NewOrderTransaction(txn, storage); + break; // Payment case PAYMENT: - return PaymentTransaction(txn, storage); - break; + return PaymentTransaction(txn, storage); + break; case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; + return OrderStatusTransaction(txn, storage); + break; case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; + return StockLevelTransaction(txn, storage); + break; case DELIVERY: - return DeliveryTransaction(txn, storage); - break; + return DeliveryTransaction(txn, storage); + break; // Invalid transaction default: - return FAILURE; - break; - } + return FAILURE; + break; + } - return FAILURE; + return FAILURE; } // The new order function is executed when the application receives a new order // transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; +int TPCC::NewOrderTransaction(TxnProto *txn, StorageManager *storage) const +{ + // First, we retrieve the warehouse from storage + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + // Next, we retrieve the district + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_write_set(0)); + assert(district->ParseFromString(*district_value)); + // Increment the district's next order ID and put to datastore + district->set_next_order_id(district->next_order_id() + 1); + assert(district->SerializeToString(district_value)); + // Not necessary since storage already has a pointer to district_value. + // storage->PutObject(district->id(), district_value); + + // Retrieve the customer we are looking for + Value *customer_value; + Customer *customer = new Customer(); + customer_value = storage->ReadObject(txn->read_set(1)); + assert(customer->ParseFromString(*customer_value)); + + // Next, we get the order line count, system time, and other args from the + // transaction proto + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int order_line_count = tpcc_args->order_line_count(0); + int order_number = tpcc_args->order_number(); + double system_time = tpcc_args->system_time(); + + // Next we create an Order object + Key order_key = txn->write_set(order_line_count + 1); + Order *order = new Order(); + order->set_id(order_key); + order->set_warehouse_id(warehouse->id()); + order->set_district_id(district->id()); + order->set_customer_id(customer->id()); + + // Set some of the auxiliary data + order->set_entry_date(system_time); + order->set_carrier_id(-1); + order->set_order_line_count(order_line_count); + order->set_all_items_local(txn->multipartition()); + + // We initialize the order line amount total to 0 + int order_line_amount_total = 0; + + for (int i = 0; i < order_line_count; i++) + { + // For each order line we parse out the three args + + string stock_key = txn->read_write_set(i + 1); + string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); + int quantity = tpcc_args->quantities(i); + + // Find the item key within the stock key + size_t item_idx = stock_key.find("i"); + string item_key = stock_key.substr(item_idx, string::npos); + + // First, we check if the item number is valid + Item *item = new Item(); + if (item_key == "i-1") + return FAILURE; + else + assert(item->ParseFromString(*GetItem(item_key))); + + // Next, we create a new order line object with std attributes + OrderLine *order_line = new OrderLine(); + Key order_line_key = txn->write_set(i); + order_line->set_order_id(order_line_key); + + // Set the attributes for this order line + order_line->set_district_id(district->id()); + order_line->set_warehouse_id(warehouse->id()); + order_line->set_number(i); + order_line->set_item_id(item_key); + order_line->set_supply_warehouse_id(supply_warehouse_key); + order_line->set_quantity(quantity); + order_line->set_delivery_date(system_time); + + // Next, we get the correct stock from the data store + Stock *stock = new Stock(); + Value *stock_value = storage->ReadObject(stock_key); + assert(stock->ParseFromString(*stock_value)); + + // Once we have it we can increase the YTD, order_count, and remote_count + stock->set_year_to_date(stock->year_to_date() + quantity); + stock->set_order_count(stock->order_count() - 1); + if (txn->multipartition()) + stock->set_remote_count(stock->remote_count() + 1); + + // And we decrease the stock's supply appropriately and rewrite to storage + if (stock->quantity() >= quantity + 10) + stock->set_quantity(stock->quantity() - quantity); + else + stock->set_quantity(stock->quantity() - quantity + 91); + + // Put the stock back into the database + assert(stock->SerializeToString(stock_value)); + // Not necessary since storage already has a ptr to stock_value. + // storage->PutObject(stock_key, stock_value); + delete stock; + + // Next, we update the order line's amount and add it to the running sum + order_line->set_amount(quantity * item->price()); + order_line_amount_total += (quantity * item->price()); + + // Finally, we write the order line to storage + Value *order_line_value = new Value(); + assert(order_line->SerializeToString(order_line_value)); + storage->PutObject(order_line_key, order_line_value); + // order->add_order_line_ptr(reinterpret_cast(order_line_value)); + + pthread_mutex_lock(&mutex_for_item); + if (storage->configuration_->this_node_id == + storage->configuration_->LookupPartition(txn->read_set(0))) + item_for_order_line[order_line_key] = StringToInt(item_key); + pthread_mutex_unlock(&mutex_for_item); + // Free memory + delete order_line; + delete item; } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; + + // We create a new NewOrder object + Key new_order_key = txn->write_set(order_line_count); + NewOrder *new_order = new NewOrder(); + new_order->set_id(new_order_key); + new_order->set_warehouse_id(warehouse->id()); + new_order->set_district_id(district->id()); + + // Serialize it and put it in the datastore + Value *new_order_value = new Value(); + assert(new_order->SerializeToString(new_order_value)); + storage->PutObject(new_order_key, new_order_value); + + // Serialize order and put it in the datastore + Value *order_value = new Value(); + assert(order->SerializeToString(order_value)); + storage->PutObject(order_key, order_value); + + if (storage->configuration_->this_node_id == + storage->configuration_->LookupPartition(txn->read_set(0))) + { + pthread_mutex_lock(&mutex_); + if (latest_order_id_for_customer.count(txn->read_set(1)) == 0) + involed_customers->push_back(txn->read_set(1)); + latest_order_id_for_customer[txn->read_set(1)] = order_key; + + if (smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) + { + smallest_order_id_for_district[txn->read_write_set(0)] = order_number; + } + order_line_number[order_key] = order_line_count; + customer_for_order[order_key] = txn->read_set(1); + + latest_order_id_for_district[txn->read_write_set(0)] = order_number; + pthread_mutex_unlock(&mutex_); + } + + // Successfully completed transaction + delete warehouse; + delete district; + delete customer; + delete order; + delete new_order; + delete tpcc_args; + return SUCCESS; } // The payment function is executed when the application receives a // payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); +int TPCC::PaymentTransaction(TxnProto *txn, StorageManager *storage) const +{ + // First, we parse out the transaction args from the TPCC proto + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int amount = tpcc_args->amount(); + + // We create a string to hold up the customer object we look up + Value *customer_value; + Key customer_key; + + // If there's a last name we do secondary keying + if (tpcc_args->has_last_name()) + { + Key secondary_key = tpcc_args->last_name(); + + // If the RW set is not at least of size 3, then no customer key was + // given to this transaction. Otherwise, we perform a check to ensure + // the secondary key we just looked up agrees with our previous lookup + if (txn->read_write_set_size() < 3 || + secondary_key != txn->read_write_set(2)) + { + // Append the newly read key to write set + if (txn->read_write_set_size() < 3) + txn->add_read_write_set(secondary_key); + + // Or the old one was incorrect so we overwrite it + else + txn->set_read_write_set(2, secondary_key); + + return REDO; + // Otherwise, we look up the customer's key + } + else + { + customer_value = storage->ReadObject(tpcc_args->last_name()); + } + + // Otherwise we use the final argument + } + else + { + customer_key = txn->read_write_set(2); + customer_value = storage->ReadObject(customer_key); } - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} + // Deserialize the warehouse object + Key warehouse_key = txn->read_write_set(0); + Value *warehouse_value = storage->ReadObject(warehouse_key); + Warehouse *warehouse = new Warehouse(); + assert(warehouse->ParseFromString(*warehouse_value)); + // Next, we update the values of the warehouse and write it out + warehouse->set_year_to_date(warehouse->year_to_date() + amount); + assert(warehouse->SerializeToString(warehouse_value)); + // Not necessary since storage already has a pointer to warehouse_value. + // storage->PutObject(warehouse_key, warehouse_value); -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} + // Deserialize the district object + Key district_key = txn->read_write_set(1); + Value *district_value = storage->ReadObject(district_key); + District *district = new District(); + assert(district->ParseFromString(*district_value)); + + // Next, we update the values of the district and write it out + district->set_year_to_date(district->year_to_date() + amount); + assert(district->SerializeToString(district_value)); + // Not necessary since storage already has a pointer to district_value. + // storage->PutObject(district_key, district_value); -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; + // We deserialize the customer + Customer *customer = new Customer(); + assert(customer->ParseFromString(*customer_value)); + + // Next, we update the customer's balance, payment and payment count + customer->set_balance(customer->balance() - amount); + customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); + customer->set_payment_count(customer->payment_count() + 1); + + // If the customer has bad credit, we update the data information attached + // to her + if (customer->credit() == "BC") + { + string data = customer->data(); + char new_information[500]; + + // Print the new_information into the buffer + snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", + customer->id().c_str(), customer->warehouse_id().c_str(), + customer->district_id().c_str(), district->id().c_str(), + warehouse->id().c_str(), amount, customer->data().c_str()); } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); + // We write the customer to disk + assert(customer->SerializeToString(customer_value)); + // Not necessary since storage already has a pointer to customer_value. + // storage->PutObject(customer_key, customer_value); + + // Finally, we create a history object and update the data + History *history = new History(); + history->set_customer_id(customer_key); + history->set_customer_warehouse_id(customer->warehouse_id()); + history->set_customer_district_id(customer->district_id()); + history->set_warehouse_id(warehouse_key); + history->set_district_id(district_key); + + // Create the data for the history object + char history_data[100]; + snprintf(history_data, sizeof(history_data), "%s %s", + warehouse->name().c_str(), district->name().c_str()); + history->set_data(history_data); - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); + // Write the history object to disk + Value *history_value = new Value(); + assert(history->SerializeToString(history_value)); + storage->PutObject(txn->write_set(0), history_value); - if(txn->read_set_size() == 1) { - delete warehouse; + // Successfully completed transaction + delete customer; + delete history; + delete district; + delete warehouse; + delete tpcc_args; return SUCCESS; - } +} + +int TPCC::OrderStatusTransaction(TxnProto *txn, StorageManager *storage) const +{ + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int order_line_count = tpcc_args->order_line_count(0); + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(1)); assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; + Value *customer_value; + Customer *customer = new Customer(); + customer_value = storage->ReadObject(txn->read_set(2)); + assert(customer->ParseFromString(*customer_value)); + + // double customer_balance = customer->balance(); + string customer_first = customer->first(); + string customer_middle = customer->middle(); + string customer_last = customer->last(); + + Order *order = new Order(); + Value *order_value = storage->ReadObject(txn->read_set(3)); assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; + // int carrier_id = order->carrier_id(); + // double entry_date = order->entry_date(); + + for (int i = 0; i < order_line_count; i++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = storage->ReadObject(txn->read_set(4 + i)); + assert(order_line->ParseFromString(*order_line_value)); + string item_key = order_line->item_id(); + string supply_warehouse_id = order_line->supply_warehouse_id(); + // int quantity = order_line->quantity(); + // double amount = order_line->amount(); + // double delivery_date = order_line->delivery_date(); + + delete order_line; } - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - + delete warehouse; delete district; - delete order; delete customer; - - } + delete order; - delete warehouse; - return SUCCESS; + return SUCCESS; } +int TPCC::StockLevelTransaction(TxnProto *txn, StorageManager *storage) const +{ + int low_stock = 0; + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int threshold = tpcc_args->threshold(); -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(1)); + assert(district->ParseFromString(*district_value)); + + int index = 0; + + int cycle = (txn->read_set_size() - 2) / 2; + for (int i = 0; i < cycle; i++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = storage->ReadObject(txn->read_set(2 + index)); + index++; + assert(order_line->ParseFromString(*order_line_value)); + string item_key = order_line->item_id(); + + Stock *stock = new Stock(); + Value *stock_value = storage->ReadObject(txn->read_set(2 + index)); + index++; + assert(stock->ParseFromString(*stock_value)); + if (stock->quantity() < threshold) + { + low_stock++; + } - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); + delete order_line; + delete stock; } - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); + delete warehouse; + delete district; + return SUCCESS; +} - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); +int TPCC::DeliveryTransaction(TxnProto *txn, StorageManager *storage) const +{ + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); - // Free storage - delete district; + if (txn->read_set_size() == 1) + { + delete warehouse; + return SUCCESS; } - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; + int delivery_district_number = txn->read_set_size() - 1; + int read_write_index = 0; + int line_count_index = 0; + for (int i = 1; i <= delivery_district_number; i++) + { + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(i)); + assert(district->ParseFromString(*district_value)); + + storage->DeleteObject(txn->read_write_set(read_write_index)); + read_write_index++; + + Order *order = new Order(); + Value *order_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(order->ParseFromString(*order_value)); + char customer_key[128]; + snprintf(customer_key, sizeof(customer_key), "%s", + order->customer_id().c_str()); + + order->set_carrier_id(rand() % 10); + + int ol_number = tpcc_args->order_line_count(line_count_index); + line_count_index++; + double total_amount = 0; + + for (int j = 0; j < ol_number; j++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(order_line->ParseFromString(*order_line_value)); + order_line->set_delivery_date(GetTime()); + total_amount = total_amount + order_line->amount(); + + delete order_line; + } + + Customer *customer = new Customer(); + Value *customer_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(customer->ParseFromString(*customer_value)); + customer->set_balance(customer->balance() + total_amount); + customer->set_delivery_count(customer->delivery_count() + 1); + + delete district; + delete order; + delete customer; } - // Free storage delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } + return SUCCESS; } -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); +// The initialize function is executed when an initialize transaction comes +// through, indicating we should populate the database with fake data +void TPCC::InitializeStorage(Storage *storage, Configuration *conf) const +{ + // We create and write out all of the warehouses + for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); + i++) + { + // First, we create a key for the warehouse + char warehouse_key[128], warehouse_key_ytd[128]; + Value *warehouse_value = new Value(); + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); + snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); + if (conf->LookupPartition(warehouse_key) != conf->this_node_id) + { + continue; + } + // Next we initialize the object and serialize it + Warehouse *warehouse = CreateWarehouse(warehouse_key); + assert(warehouse->SerializeToString(warehouse_value)); - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(warehouse_key) == conf->this_node_id) + { + storage->PutObject(warehouse_key, warehouse_value); + storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); + } - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); + // Next, we create and write out all of the districts + for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) + { + // First, we create a key for the district + char district_key[128], district_key_ytd[128]; + snprintf(district_key, sizeof(district_key), "w%dd%d", i, j); + snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", i, j); + + // Next we initialize the object and serialize it + Value *district_value = new Value(); + District *district = CreateDistrict(district_key, warehouse_key); + assert(district->SerializeToString(district_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(district_key) == conf->this_node_id) + { + storage->PutObject(district_key, district_value); + storage->PutObject(district_key_ytd, new Value(*district_value)); + } + + // Next, we create and write out all of the customers + for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) + { + // First, we create a key for the customer + char customer_key[128]; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", i, j, k); + + // Next we initialize the object and serialize it + Value *customer_value = new Value(); + Customer *customer = + CreateCustomer(customer_key, district_key, warehouse_key); + assert(customer->SerializeToString(customer_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(customer_key) == conf->this_node_id) + storage->PutObject(customer_key, customer_value); + delete customer; + } + + // Free storage + delete district; + } - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); + // Next, we create and write out all of the stock + for (int j = 0; j < NUMBER_OF_ITEMS; j++) + { + // First, we create a key for the stock + char item_key[128]; + Value *stock_value = new Value(); + snprintf(item_key, sizeof(item_key), "i%d", j); + + // Next we initialize the object and serialize it + Stock *stock = CreateStock(item_key, warehouse_key); + assert(stock->SerializeToString(stock_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(stock->id()) == conf->this_node_id) + storage->PutObject(stock->id(), stock_value); + delete stock; + } + + // Free storage + delete warehouse; + } - return warehouse; + // Finally, all the items are initialized + srand(1000); + for (int i = 0; i < NUMBER_OF_ITEMS; i++) + { + // First, we create a key for the item + char item_key[128]; + Value *item_value = new Value(); + snprintf(item_key, sizeof(item_key), "i%d", i); + + // Next we initialize the object and serialize it + Item *item = CreateItem(item_key); + assert(item->SerializeToString(item_value)); + + // Finally, we pass it off to the local record of items + SetItem(string(item_key), item_value); + delete item; + } } -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); +// The following method is a dumb constructor for the warehouse protobuffer +Warehouse *TPCC::CreateWarehouse(Key warehouse_key) const +{ + Warehouse *warehouse = new Warehouse(); + + // We initialize the id and the name fields + warehouse->set_id(warehouse_key); + warehouse->set_name(RandomString(10)); + + // Provide some information to make TPC-C happy + warehouse->set_street_1(RandomString(20)); + warehouse->set_street_2(RandomString(20)); + warehouse->set_city(RandomString(20)); + warehouse->set_state(RandomString(2)); + warehouse->set_zip(RandomString(9)); + + // Set default financial information + warehouse->set_tax(0.05); + warehouse->set_year_to_date(0.0); + + return warehouse; +} - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); +District *TPCC::CreateDistrict(Key district_key, Key warehouse_key) const +{ + District *district = new District(); - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); + // We initialize the id and the name fields + district->set_id(district_key); + district->set_warehouse_id(warehouse_key); + district->set_name(RandomString(10)); - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); + // Provide some information to make TPC-C happy + district->set_street_1(RandomString(20)); + district->set_street_2(RandomString(20)); + district->set_city(RandomString(20)); + district->set_state(RandomString(2)); + district->set_zip(RandomString(9)); - return district; + // Set default financial information + district->set_tax(0.05); + district->set_year_to_date(0.0); + district->set_next_order_id(1); + + return district; } -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; +Customer *TPCC::CreateCustomer(Key customer_key, + Key district_key, + Key warehouse_key) const +{ + Customer *customer = new Customer(); + + // We initialize the various keys + customer->set_id(customer_key); + customer->set_district_id(district_key); + customer->set_warehouse_id(warehouse_key); + + // Next, we create a first and middle name + customer->set_first(RandomString(20)); + customer->set_middle(RandomString(20)); + customer->set_last(customer_key); + + // Provide some information to make TPC-C happy + customer->set_street_1(RandomString(20)); + customer->set_street_2(RandomString(20)); + customer->set_city(RandomString(20)); + customer->set_state(RandomString(2)); + customer->set_zip(RandomString(9)); + + // Set default financial information + customer->set_since(0); + customer->set_credit("GC"); + customer->set_credit_limit(0.01); + customer->set_discount(0.5); + customer->set_balance(0); + customer->set_year_to_date_payment(0); + customer->set_payment_count(0); + customer->set_delivery_count(0); + + // Set some miscellaneous data + customer->set_data(RandomString(50)); + + return customer; } -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); +Stock *TPCC::CreateStock(Key item_key, Key warehouse_key) const +{ + Stock *stock = new Stock(); - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); + // We initialize the various keys + char stock_key[128]; + snprintf(stock_key, sizeof(stock_key), "%ss%s", warehouse_key.c_str(), + item_key.c_str()); + stock->set_id(stock_key); + stock->set_warehouse_id(warehouse_key); + stock->set_item_id(item_key); - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); + // Next, we create a first and middle name + stock->set_quantity(rand() % 100 + 100); - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); + // Set default financial information + stock->set_year_to_date(0); + stock->set_order_count(0); + stock->set_remote_count(0); - // Set some miscellaneous data - stock->set_data(RandomString(50)); + // Set some miscellaneous data + stock->set_data(RandomString(50)); - return stock; + return stock; } -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); +Item *TPCC::CreateItem(Key item_key) const +{ + Item *item = new Item(); - // We initialize the item's key - item->set_id(item_key); + // We initialize the item's key + item->set_id(item_key); - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); + // Initialize some fake data for the name, price and data + item->set_name(RandomString(24)); + item->set_price(rand() % 100); + item->set_data(RandomString(50)); - return item; -} + return item; +} \ No newline at end of file diff --git a/src_calvin/applications/tpcc.h b/src_calvin/applications/tpcc.h index b01657e2..6e63d34b 100644 --- a/src_calvin/applications/tpcc.h +++ b/src_calvin/applications/tpcc.h @@ -12,12 +12,18 @@ #include "proto/txn.pb.h" #include "common/configuration.h" -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 +// #define WAREHOUSES_PER_NODE 12 +// #define DISTRICTS_PER_WAREHOUSE 10 #define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 +// #define CUSTOMERS_PER_DISTRICT 3000 #define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 +// #define NUMBER_OF_ITEMS 100000 + +#define WAREHOUSES_PER_NODE 1 +#define DISTRICTS_PER_WAREHOUSE 2 +#define CUSTOMERS_PER_DISTRICT 100 +#define NUMBER_OF_ITEMS 1000 +#define ORDERS_PER_DISTRICT 100 using std::string; @@ -27,94 +33,118 @@ class Customer; class Item; class Stock; -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); +class TPCC : public Application +{ +public: + enum TxnType + { + INITIALIZE = 0, + NEW_ORDER = 1, + PAYMENT = 2, + ORDER_STATUS = 3, + DELIVERY = 4, + STOCK_LEVEL = 5, + }; + + virtual ~TPCC() {} + + // Load generator for a new transaction + virtual TxnProto *NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration *config) const; + + // The key converter takes a valid key (string) and converts it to an id + // for the checkpoint to use + static int CheckpointID(Key key) + { + // Initial dissection of the key + size_t id_idx; + + // Switch based on key type + size_t bad = string::npos; + if ((id_idx = key.find("s")) != bad) + { + size_t ware = key.find("w"); + return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("c")) != bad) + { + return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) + { + return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("w")) != bad) + { + return atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("i")) != bad) + { + return 3000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("ol")) != bad) + { + return 4000000 + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("no")) != bad) + { + return 5000000 + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("o")) != bad) + { + return 6000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("h")) != bad) + { + return 7000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("ln")) != bad) + { + return 8000000 + atoi(&key[id_idx + 2]); + } + + // Invalid key + return -1; } - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; + // Simple execution of a transaction using a given storage + virtual int Execute(TxnProto *txn, StorageManager *storage) const; -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; + /* TODO(Thad): Uncomment once testing friend class exists + private: */ + // When the first transaction is called, the following function initializes + // a set of fake data for use in the application + virtual void InitializeStorage(Storage *storage, Configuration *conf) const; - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; + // The following methods are simple randomized initializers that provide us + // fake data for our TPC-C function + Warehouse *CreateWarehouse(Key id) const; + District *CreateDistrict(Key id, Key warehouse_id) const; + Customer *CreateCustomer(Key id, Key district_id, Key warehouse_id) const; + Item *CreateItem(Key id) const; + Stock *CreateStock(Key id, Key warehouse_id) const; - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; + // A NewOrder call takes a set of args and a transaction id and performs + // the new order transaction as specified by TPC-C. The return is 1 for + // success or 0 for failure. + int NewOrderTransaction(TxnProto *txn, StorageManager *storage) const; - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; + // A Payment call takes a set of args as the parameter and performs the + // payment transaction, returning a 1 for success or 0 for failure. + int PaymentTransaction(TxnProto *txn, StorageManager *storage) const; - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; + int OrderStatusTransaction(TxnProto *txn, StorageManager *storage) const; - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; + int StockLevelTransaction(TxnProto *txn, StorageManager *storage) const; - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; + int DeliveryTransaction(TxnProto *txn, StorageManager *storage) const; - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; + // The following are implementations of retrieval and writing for local items + Value *GetItem(Key key) const; + void SetItem(Key key, Value *value) const; }; -#endif // _DB_APPLICATIONS_TPCC_H_ +#endif // _DB_APPLICATIONS_TPCC_H_ \ No newline at end of file diff --git a/src_calvin/applications/ycsb.cc b/src_calvin/applications/ycsb.cc new file mode 100644 index 00000000..7ffb7b4a --- /dev/null +++ b/src_calvin/applications/ycsb.cc @@ -0,0 +1,283 @@ +// FILE: applications/ycsb.cc + +#include "applications/ycsb.h" + +#include +#include +#include +#include +#include +#include + +#include "backend/storage.h" +#include "backend/storage_manager.h" +#include "common/configuration.h" +#include "common/definitions.hh" +#include "common/random.hh" +#include "common/utils.h" +#include "proto/txn.pb.h" + +enum YCSBTxnType +{ + YCSB_TXN_SP, + YCSB_TXN_MP, + YCSB_TXN_INITIALIZE +}; + +// ---- 補助: Random から [0,1) 一様乱数を作る(Random::Uniform のみで実装)---- +static inline double U01(Random &rnd) +{ + // 2^30 で割ることで [0,1) の double を得る + const uint32_t M = 1u << 30; + return static_cast(rnd.Uniform(M)) / static_cast(M); +} + +// ---- 補助: 簡易 Zipf サンプラー(n が小さいときに使う想定。hot_records_ 用)---- +// 返り値は [0, n-1] +static uint64_t ZipfSample(Random &rnd, uint64_t n, double theta) +{ + if (n <= 1 || theta <= 0.0) + return 0; + + // 定数の前計算(n が小さいので毎回でも軽い) + double zetan = 0.0; + for (uint64_t i = 1; i <= n; ++i) + { + zetan += 1.0 / std::pow(static_cast(i), theta); + } + const double zeta2 = 1.0 + std::pow(2.0, -theta); + const double alpha = 1.0 / (1.0 - theta); + const double eta = (1.0 - std::pow(2.0 / static_cast(n), 1.0 - theta)) / + (1.0 - zeta2 / zetan); + + const double u = U01(rnd); + const double uz = u * zetan; + + if (uz < 1.0) + return 0; + if (uz < 1.0 + std::pow(0.5, theta)) + return 1; + + // 1-origin を想定した式なので 0-origin に揃える + const double val = std::floor( + static_cast(n) * std::pow(eta * u - eta + 1.0, alpha)); + // 範囲ガード + if (val < 0.0) + return 0; + if (val >= static_cast(n)) + return n - 1; + return static_cast(val); +} + +YCSB::YCSB(double read_ratio, double skew, uint64 db_size, uint64 hot_records) + : read_ratio_(read_ratio), + skew_(skew), + db_size_(db_size), + hot_records_(hot_records), + rnd_() {} + +void YCSB::InitializeStorage(Storage *storage, Configuration *config) const +{ + for (uint64 i = 0; i < db_size_; i++) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", i); + if (config->LookupPartition(key_str) == config->this_node_id) + { + Value *value = new Value(RandomString(128), 0); + storage->PutObject(key_str, value); + } + } + // 事前にキー候補を前計算(遅延初期化でも可) + const_cast(this)->PrecomputeKeys(config); +} + +// 事前計算:各パーティションに属する「非ホット」キーの番号リストを作成 +void YCSB::PrecomputeKeys(Configuration *config) +{ + keys_per_partition_.clear(); + keys_per_partition_.resize(config->all_nodes.size()); + for (uint64_t i = hot_records_; i < db_size_; ++i) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", i); + int partition_id = config->LookupPartition(key_str); + keys_per_partition_[partition_id].push_back(i); + } + keys_ready_.store(true, std::memory_order_release); + std::cout << "Key precomputation complete." << std::endl; +} + +TxnProto *YCSB::NewTxn(int64 txn_id, int /*txn_type*/, std::string /*args*/, + Configuration *config) const +{ + // 単一ノードなら常にSP。マルチノード時は 1% を MP に(従来仕様) + const double mp_ratio = 0.01; + if (config->all_nodes.size() == 1 || U01(rnd_) >= mp_ratio) + { + uint32 part = rnd_.Uniform(static_cast(config->all_nodes.size())); + return YCSBTxnSP(txn_id, part, config); + } + else + { + uint32 part1 = rnd_.Uniform(static_cast(config->all_nodes.size())); + uint32 part2; + do + { + part2 = rnd_.Uniform(static_cast(config->all_nodes.size())); + } while (part1 == part2); + return YCSBTxnMP(txn_id, part1, part2, config); + } +} + +int YCSB::Execute(TxnProto * /*txn*/, StorageManager * /*storage*/) const +{ + // Calvin の YCSB は実行ロジックが StorageManager 経由で処理されるためここは空 + return 0; +} + +// ランダムキー抽選(前計算リストから重複ナシで取り出し) +void YCSB::GetRandomKeys(std::set &keys, int num_keys, uint32 part) const +{ + keys.clear(); + if (!keys_ready_.load(std::memory_order_acquire)) + return; + const auto &key_candidates = keys_per_partition_[part]; + if (key_candidates.empty()) + return; + + while ((int)keys.size() < num_keys) + { + uint64_t index = rnd_.Uniform(static_cast(key_candidates.size())); + keys.insert(key_candidates[index]); + } +} + +TxnProto *YCSB::YCSBTxnSP(int64 txn_id, uint32 part, Configuration *config) const +{ + if (!keys_ready_.load(std::memory_order_acquire)) + const_cast(this)->PrecomputeKeys(config); + + TxnProto *txn = new TxnProto(); + txn->set_txn_id(txn_id); + txn->set_txn_type(YCSB_TXN_SP); + + const bool is_read = (U01(rnd_) < read_ratio_); + + // --- ホットキー 1本(Zipf or 一様)--- + uint64 hot_idx = 0; + if (skew_ > 0.0) + { + hot_idx = ZipfSample(rnd_, static_cast(hot_records_), skew_); + } + else + { + hot_idx = rnd_.Uniform(static_cast(hot_records_)); + } + char hotkey_str[32]; + snprintf(hotkey_str, sizeof(hotkey_str), "k%lu", hot_idx); + if (is_read) + { + txn->add_read_set(hotkey_str); + } + else + { + txn->add_read_write_set(hotkey_str); + } + + // --- コールド側(パーティションに属する候補から一様抽選、重複なし)--- + std::set keys; + GetRandomKeys(keys, RW_SET_SIZE - 1, part); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + return txn; +} + +TxnProto *YCSB::YCSBTxnMP(int64 txn_id, uint32 part1, uint32 part2, + Configuration *config) const +{ + if (!keys_ready_.load(std::memory_order_acquire)) + const_cast(this)->PrecomputeKeys(config); + + TxnProto *txn = new TxnProto(); + txn->set_txn_id(txn_id); + txn->set_txn_type(YCSB_TXN_MP); + txn->set_multipartition(true); + + const bool is_read = (U01(rnd_) < read_ratio_); + + // --- 各パーティションのホットキー(Zipf or 一様)--- + uint64 hot1 = (skew_ > 0.0) + ? ZipfSample(rnd_, static_cast(hot_records_), skew_) + : rnd_.Uniform(static_cast(hot_records_)); + uint64 hot2 = (skew_ > 0.0) + ? ZipfSample(rnd_, static_cast(hot_records_), skew_) + : rnd_.Uniform(static_cast(hot_records_)); + + char hotkey1_str[32], hotkey2_str[32]; + snprintf(hotkey1_str, sizeof(hotkey1_str), "k%lu", hot1); + snprintf(hotkey2_str, sizeof(hotkey2_str), "k%lu", hot2); + if (is_read) + { + txn->add_read_set(hotkey1_str); + txn->add_read_set(hotkey2_str); + } + else + { + txn->add_read_write_set(hotkey1_str); + txn->add_read_write_set(hotkey2_str); + } + + // --- コールド側:各パーティションから一様抽選 --- + std::set keys; + GetRandomKeys(keys, RW_SET_SIZE / 2 - 1, part1); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + + keys.clear(); + GetRandomKeys(keys, RW_SET_SIZE / 2 - 1, part2); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + + return txn; +} + +TxnProto *YCSB::InitializeTxn() const +{ + TxnProto *txn = new TxnProto(); + txn->set_txn_type(YCSB_TXN_INITIALIZE); + return txn; +} diff --git a/src_calvin/applications/ycsb.h b/src_calvin/applications/ycsb.h new file mode 100644 index 00000000..d1f94ae1 --- /dev/null +++ b/src_calvin/applications/ycsb.h @@ -0,0 +1,47 @@ +#ifndef _DB_APPLICATIONS_YCSB_H_ +#define _DB_APPLICATIONS_YCSB_H_ + +#include "applications/application.h" +#include "common/random.hh" // ← Random の定義 +#include +#include +#include +#include + +class YCSB : public Application +{ +public: + YCSB(double read_ratio, double skew, uint64 db_size, uint64 hot_records); + + void InitializeStorage(Storage *storage, Configuration *config) const override; + TxnProto *NewTxn(int64 txn_id, int txn_type, std::string args, Configuration *config) const override; + int Execute(TxnProto *txn, StorageManager *storage) const override; + + // Application の宣言と一致しない環境があるため override は付けない + TxnProto *InitializeTxn() const; + + // 初期化時または遅延でキー候補を前計算 + void PrecomputeKeys(Configuration *config); + +private: + TxnProto *YCSBTxnSP(int64 txn_id, uint32 part, Configuration *config) const; + TxnProto *YCSBTxnMP(int64 txn_id, uint32 part1, uint32 part2, Configuration *config) const; + + // part に属するランダムキーを重複なしで num_keys 個選ぶ + void GetRandomKeys(std::set &keys, int num_keys, uint32 part) const; + + // パラメータ + double read_ratio_; + double skew_; // ← 追加:Zipf 用スキュー (0 なら一様) + uint64 db_size_; + uint64 hot_records_; + + // 乱数生成器(common/random.hh) + mutable Random rnd_; + + // 前計算キャッシュ + mutable std::atomic keys_ready_{false}; + std::vector> keys_per_partition_; +}; + +#endif // _DB_APPLICATIONS_YCSB_H_ diff --git a/src_calvin/backend/collapsed_versioned_storage.cc b/src_calvin/backend/collapsed_versioned_storage.cc index aa0f99f9..6a7c8ca6 100644 --- a/src_calvin/backend/collapsed_versioned_storage.cc +++ b/src_calvin/backend/collapsed_versioned_storage.cc @@ -14,13 +14,13 @@ using std::string; #ifdef TPCCHACK #define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; +// For inserted objects we need to make a representation that is thread-safe +// (i.e. an array). This is kind of hacky, but since this only corresponds +// to TPCC, we'll be okay +Value* NewOrderStore[MAXARRAYSIZE]; +Value* OrderStore[MAXARRAYSIZE]; +Value* OrderLineStore[MAXARRAYSIZE * 15]; +Value* HistoryStore[MAXARRAYSIZE]; #endif Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { @@ -36,14 +36,14 @@ Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { } else { #endif - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; + // Check to see if a match even exists + if (objects_.count(key) != 0) { + for (DataNode* list = objects_[key]; list; list = list->next) { + if (list->txn_id <= txn_id) { + return list->value; + } } } - } #ifdef TPCCHACK } @@ -53,12 +53,12 @@ Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { return NULL; } -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, +bool CollapsedVersionedStorage::PutObject(const Key& key, + Value* value, int64 txn_id) { #ifdef TPCCHACK if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; + OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = value; } else if (key.find("no") != string::npos) { NewOrderStore[txn_id] = value; } else if (key.find("o") != string::npos) { @@ -68,26 +68,26 @@ bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, } else { #endif - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; + // Create the new version to insert into the list + DataNode* item = new DataNode(); + item->txn_id = txn_id; + item->value = value; + item->next = NULL; + + // Is the most recent value a candidate for pruning? + DataNode* current; + if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { + int64 most_recent = current->txn_id; + + if ((most_recent > stable_ && txn_id > stable_) || + (most_recent <= stable_ && txn_id <= stable_)) { + item->next = current->next; + delete current; + } else { + item->next = current; + } } - } - objects_[key] = item; + objects_[key] = item; #ifdef TPCCHACK } @@ -117,12 +117,12 @@ bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { objects_[key]->txn_id = txn_id; objects_[key]->value = NULL; - // Otherwise we need to free the head + // Otherwise we need to free the head } else if (list != NULL && objects_[key] == list) { delete objects_[key]; objects_[key] = NULL; - // Lastly, we may only want to free the tail + // Lastly, we may only want to free the tail } else if (list != NULL) { delete list; objects_[key]->next = NULL; @@ -133,8 +133,8 @@ bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { int CollapsedVersionedStorage::Checkpoint() { pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); + int thread_status = + pthread_create(&checkpointing_daemon, NULL, &RunCheckpointer, this); return thread_status; } @@ -159,17 +159,13 @@ void CollapsedVersionedStorage::CaptureCheckpoint() { // Write to disk int key_length = key.length(); int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); + fprintf( + checkpoint, "%c%c%c%c%s%c%c%c%c%s", static_cast(key_length >> 24), + static_cast(key_length >> 16), static_cast(key_length >> 8), + static_cast(key_length), key.c_str(), + static_cast(val_length >> 24), + static_cast(val_length >> 16), static_cast(val_length >> 8), + static_cast(val_length), result->c_str()); // Remove object from tree if there's an old version if (it->second->next != NULL) diff --git a/src_calvin/backend/fetching_storage.cc b/src_calvin/backend/fetching_storage.cc index f5514372..e4cedc3e 100644 --- a/src_calvin/backend/fetching_storage.cc +++ b/src_calvin/backend/fetching_storage.cc @@ -4,6 +4,7 @@ // main memory, disk, and swapping algorithms. #include "backend/fetching_storage.h" +#include "common/definitions.hh" typedef FetchingStorage::Latch Latch; @@ -26,8 +27,7 @@ FetchingStorage::FetchingStorage() { // 1 MILLION LATCHES! latches_ = new Latch[1000000]; - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); + pthread_create(&gc_thread_, NULL, RunGCThread, reinterpret_cast(this)); } FetchingStorage::~FetchingStorage() { @@ -45,22 +45,22 @@ Latch* FetchingStorage::LatchFor(const Key& key) { } void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); + char path[255]; + snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); + char key_c_str[255]; + memset(&key_c_str, 0, 255); + readlink(path, key_c_str, 255); + *key = string((strrchr(key_c_str, '/') + 1)); } -void* FetchingStorage::RunGCThread(void *arg) { +void* FetchingStorage::RunGCThread(void* arg) { FetchingStorage* storage = reinterpret_cast(arg); while (true) { double start_time = GetTime(); for (int i = COLD_CUTOFF; i < 1000000; i++) { storage->HardUnfetch(IntToString(i)); } - usleep(static_cast(1000000*(GetTime()-start_time))); + usleep(static_cast(1000000 * (GetTime() - start_time))); } return NULL; } @@ -77,7 +77,8 @@ Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { assert(latch->active_requests > 0); pthread_mutex_unlock(&latch->lock_); // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} + while (latch->state == FETCHING) { + } return main_memory_->ReadObject(key); } @@ -157,7 +158,7 @@ bool FetchingStorage::HardUnfetch(const Key& key) { if (active_requests == 0 && previous_state == IN_MEMORY) { Value* result = main_memory_->ReadObject(key); int len = strlen(result->c_str()); - char* c_result = new char[len+1]; + char* c_result = new char[len + 1]; strcpy(c_result, result->c_str()); return FilePut(key, c_result, len); } else { @@ -182,8 +183,8 @@ bool FetchingStorage::Unfetch(const Key& key) { ///////////////// Asynchronous Callbacks //////////////////////// void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; + struct aiocb* req; + req = (struct aiocb*)sigval.sival_ptr; /* Did the request complete? */ if (aio_error(req) == 0) { /* Request completed successfully, get the return status */ @@ -209,8 +210,8 @@ void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { } void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; + struct aiocb* req; + req = (struct aiocb*)sigval.sival_ptr; /* Did the request complete? */ if (aio_error(req) == 0) { /* Request completed successfully, get the return status */ @@ -225,8 +226,8 @@ void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { State state = latch->state; pthread_mutex_unlock(&latch->lock_); if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; + store->main_memory_->DeleteObject(key); + latch->state = ON_DISK; } close(req->aio_fildes); delete[] reinterpret_cast(req->aio_buf); @@ -242,7 +243,7 @@ void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { bool FetchingStorage::FileRead(const Key& key, char* result, int size) { string fileName(STORAGE_PATH); fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); + int fd = open(fileName.c_str(), O_RDONLY | O_NONBLOCK); if (fd == -1) return false; return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; @@ -252,14 +253,17 @@ bool FetchingStorage::FilePut(const Key& key, char* value, int size) { string fileName(STORAGE_PATH); fileName.append(key); mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); + int fd = + open(fileName.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_NONBLOCK, mode); if (fd == -1) return false; return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; } -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { +aiocb* FetchingStorage::generateControlBlock(int fd, + char* buf, + const int size, + Operation op) { aiocb* aiocbp = new aiocb(); aiocbp->aio_fildes = fd; aiocbp->aio_offset = 0; @@ -269,11 +273,9 @@ aiocb* FetchingStorage::generateControlBlock( /* Link the AIO request with a thread callback */ aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; + aiocbp->aio_sigevent.sigev_notify_function = &PrefetchCompletionHandler; else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; + aiocbp->aio_sigevent.sigev_notify_function = &UnfetchCompletionHandler; aiocbp->aio_sigevent.sigev_notify_attributes = NULL; aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; return aiocbp; diff --git a/src_calvin/backend/fetching_storage.h b/src_calvin/backend/fetching_storage.h index 273dc04c..c63541a0 100644 --- a/src_calvin/backend/fetching_storage.h +++ b/src_calvin/backend/fetching_storage.h @@ -15,6 +15,8 @@ #include #include +#include + #include "common/utils.h" #include "backend/storage.h" #include "backend/simple_storage.h" @@ -22,8 +24,6 @@ #define PAGE_SIZE 1000 #define STORAGE_PATH "../db/storage/" -#define COLD_CUTOFF 990000 - class FetchingStorage : public Storage { public: static FetchingStorage* BuildStorage(); @@ -31,15 +31,13 @@ class FetchingStorage : public Storage { virtual Value* ReadObject(const Key& key, int64 txn_id = 0); virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); + virtual bool Prefetch(const Key& key, double* wait_time); + virtual bool Unfetch(const Key& key); bool HardUnfetch(const Key& key); // Latch object that stores a counter for readlocks and a boolean for write // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; + enum State { UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING }; class Latch { public: int active_requests; // Can be as many as you want. @@ -52,7 +50,7 @@ class FetchingStorage : public Storage { pthread_mutex_init(&lock_, NULL); } }; - Latch* LatchFor(const Key &key); + Latch* LatchFor(const Key& key); static void PrefetchCompletionHandler(sigval_t sigval); static void UnfetchCompletionHandler(sigval_t sigval); static void GetKey(int fd, Key* key); @@ -65,9 +63,7 @@ class FetchingStorage : public Storage { * is being placed here. After November 1, we can swap out backends. */ - enum Operation { - FETCH, RELEASE - }; + enum Operation { FETCH, RELEASE }; // Registers an asynchronous read. bool FileRead(const Key& key, char* result, int size); @@ -87,7 +83,7 @@ class FetchingStorage : public Storage { Latch* latches_; // GC thread stuff. - static void* RunGCThread(void *arg); + static void* RunGCThread(void* arg); pthread_t gc_thread_; }; #endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_calvin/backend/simple_storage.cc b/src_calvin/backend/simple_storage.cc index 46237c8f..73196601 100644 --- a/src_calvin/backend/simple_storage.cc +++ b/src_calvin/backend/simple_storage.cc @@ -14,9 +14,9 @@ Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { } bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); + pthread_mutex_lock(&mutex_); objects_[key] = value; -pthread_mutex_unlock(&mutex_); + pthread_mutex_unlock(&mutex_); return true; } diff --git a/src_calvin/backend/simple_storage.h b/src_calvin/backend/simple_storage.h index f15f69e9..00fffe67 100644 --- a/src_calvin/backend/simple_storage.h +++ b/src_calvin/backend/simple_storage.h @@ -19,8 +19,8 @@ class SimpleStorage : public Storage { virtual ~SimpleStorage() {} // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } + virtual bool Prefetch(const Key& key, double* wait_time) { return false; } + virtual bool Unfetch(const Key& key) { return false; } virtual Value* ReadObject(const Key& key, int64 txn_id = 0); virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); virtual bool DeleteObject(const Key& key, int64 txn_id = 0); @@ -32,7 +32,5 @@ class SimpleStorage : public Storage { private: unordered_map objects_; pthread_mutex_t mutex_; - }; #endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_calvin/backend/storage.h b/src_calvin/backend/storage.h index fa2e4efb..cbbc48bf 100644 --- a/src_calvin/backend/storage.h +++ b/src_calvin/backend/storage.h @@ -19,11 +19,11 @@ class Storage { // Loads object specified by 'key' into memory if currently stored // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; + virtual bool Prefetch(const Key& key, double* wait_time) = 0; // Unfetch object on memory, writing it off to disk, asynchronously or // otherwise. - virtual bool Unfetch(const Key &key) = 0; + virtual bool Unfetch(const Key& key) = 0; // If the object specified by 'key' exists, copies the object into '*result' // and returns true. If the object does not exist, false is returned. @@ -46,4 +46,3 @@ class Storage { }; #endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_calvin/backend/storage_manager.cc b/src_calvin/backend/storage_manager.cc index 7de4ff40..154c4cfc 100644 --- a/src_calvin/backend/storage_manager.cc +++ b/src_calvin/backend/storage_manager.cc @@ -12,10 +12,14 @@ #include "proto/txn.pb.h" #include "proto/message.pb.h" -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { +StorageManager::StorageManager(Configuration* config, + Connection* connection, + Storage* actual_storage, + TxnProto* txn) + : configuration_(config), + connection_(connection), + actual_storage_(actual_storage), + txn_(txn) { MessageProto message; // If reads are performed at this node, execute local reads and broadcast @@ -111,4 +115,3 @@ bool StorageManager::DeleteObject(const Key& key) { else return true; // Not this node's problem. } - diff --git a/src_calvin/backend/storage_manager.h b/src_calvin/backend/storage_manager.h index a374017a..67424795 100644 --- a/src_calvin/backend/storage_manager.h +++ b/src_calvin/backend/storage_manager.h @@ -40,8 +40,10 @@ class TxnProto; class StorageManager { public: // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); + StorageManager(Configuration* config, + Connection* connection, + Storage* actual_storage, + TxnProto* txn); ~StorageManager(); @@ -58,7 +60,7 @@ class StorageManager { // this node. bool writer; -// private: + // private: friend class DeterministicScheduler; // Pointer to the configuration object for this node. @@ -80,8 +82,6 @@ class StorageManager { unordered_map objects_; vector remote_reads_; - }; #endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_calvin/backend/versioned_storage.h b/src_calvin/backend/versioned_storage.h index 24c6223b..45c4ee35 100644 --- a/src_calvin/backend/versioned_storage.h +++ b/src_calvin/backend/versioned_storage.h @@ -37,10 +37,9 @@ class VersionedStorage : public Storage { // specified (i.e. txn_id). virtual int Checkpoint() = 0; - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } + virtual bool Prefetch(const Key& key, double* wait_time) { return false; } + virtual bool Unfetch(const Key& key) { return false; } }; #endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_calvin/common/configuration.cc b/src_calvin/common/configuration.cc index 552ec8ce..af1722f8 100644 --- a/src_calvin/common/configuration.cc +++ b/src_calvin/common/configuration.cc @@ -34,17 +34,12 @@ int Configuration::LookupPartition(const Key& key) const { bool Configuration::WriteToFile(const string& filename) const { FILE* fp = fopen(filename.c_str(), "w"); if (fp == NULL) - return false; + return false; for (map::const_iterator it = all_nodes.begin(); it != all_nodes.end(); ++it) { Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); + fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", it->first, node->replica_id, + node->partition_id, node->cores, node->host.c_str(), node->port); } fclose(fp); return true; @@ -88,11 +83,11 @@ void Configuration::ProcessConfigLine(char key[], char value[]) { // Parse additional node addributes. char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); + node->replica_id = atoi(strtok_r(value, ":", &tok)); node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); + node->cores = atoi(strtok_r(NULL, ":", &tok)); + const char* host = strtok_r(NULL, ":", &tok); + node->port = atoi(strtok_r(NULL, ":", &tok)); // Translate hostnames to IP addresses. string ip; @@ -105,9 +100,8 @@ void Configuration::ProcessConfigLine(char key[], char value[]) { char buf[32]; memmove(&n, ent->h_addr_list[0], ent->h_length); n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); + snprintf(buf, sizeof(buf), "%u.%u.%u.%u", n >> 24, (n >> 16) & 0xff, + (n >> 8) & 0xff, n & 0xff); ip = buf; } } @@ -116,4 +110,3 @@ void Configuration::ProcessConfigLine(char key[], char value[]) { all_nodes[node->node_id] = node; } } - diff --git a/src_calvin/common/configuration.h b/src_calvin/common/configuration.h index 975e7355..9712604b 100644 --- a/src_calvin/common/configuration.h +++ b/src_calvin/common/configuration.h @@ -28,10 +28,8 @@ #include #include - #include "common/types.h" - using std::map; using std::string; using std::vector; @@ -93,4 +91,3 @@ class Configuration { }; #endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_calvin/common/connection.cc b/src_calvin/common/connection.cc index d6a1e7d5..98073793 100644 --- a/src_calvin/common/connection.cc +++ b/src_calvin/common/connection.cc @@ -8,12 +8,17 @@ #include "common/configuration.h" #include "common/utils.h" +#include "common/definitions.hh" +#include "common/debug.hh" using zmq::socket_t; ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { + : configuration_(config), + context_(1), + new_connection_channel_(NULL), + delete_connection_channel_(NULL), + deconstructor_invoked_(false) { // Lookup port. (Pick semi-arbitrary port if node id < 0). if (config->this_node_id < 0) port_ = config->all_nodes.begin()->second->port; @@ -33,7 +38,7 @@ ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) // Wait for other nodes to bind sockets before connecting to them. Spin(0.1); -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; + send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; // Connect to remote outgoing sockets. for (map::const_iterator it = config->all_nodes.begin(); @@ -47,22 +52,19 @@ send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; } } -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + cpu_set_t cpuset; + pthread_attr_t attr; + pthread_attr_init(&attr); + // pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); + CPU_ZERO(&cpuset); + CPU_SET(MULTIPLEXER_CORE, &cpuset); + pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); + pthread_create(&thread_, &attr, RunMultiplexer, + reinterpret_cast(this)); // Initialize mutex for future calls to NewConnection. pthread_mutex_init(&new_connection_mutex_, NULL); @@ -92,13 +94,15 @@ ConnectionMultiplexer::~ConnectionMultiplexer() { it != inproc_out_.end(); ++it) { delete it->second; } - - for (unordered_map*>::iterator it = remote_result_.begin(); + + for (unordered_map*>::iterator it = + remote_result_.begin(); it != remote_result_.end(); ++it) { delete it->second; } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); + + for (unordered_map*>::iterator it = + link_unlink_queue_.begin(); it != link_unlink_queue_.end(); ++it) { delete it->second; } @@ -113,7 +117,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel) { // Wait for the Run() loop to create the Connection object. (It will reset // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} + while (new_connection_channel_ != NULL) { + } Connection* connection = new_connection_; new_connection_ = NULL; @@ -124,7 +129,9 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel) { return connection; } -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { +Connection* ConnectionMultiplexer::NewConnection( + const string& channel, + AtomicQueue** aa) { // Disallow concurrent calls to NewConnection/~Connection. pthread_mutex_lock(&new_connection_mutex_); remote_result_[channel] = *aa; @@ -133,7 +140,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQu // Wait for the Run() loop to create the Connection object. (It will reset // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} + while (new_connection_channel_ != NULL) { + } Connection* connection = new_connection_; new_connection_ = NULL; @@ -144,6 +152,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQu } void ConnectionMultiplexer::Run() { + PrintCpu("Multiplexer", 0); + MessageProto message; zmq::message_t msg; @@ -154,7 +164,8 @@ void ConnectionMultiplexer::Run() { // Channel name already in use. Report an error and set new_connection_ // (which NewConnection() will return) to NULL. std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; + << (*new_connection_channel_) << "\n" + << std::flush; new_connection_ = NULL; } else { // Channel name is not already in use. Create a new Connection object @@ -171,10 +182,10 @@ void ConnectionMultiplexer::Run() { new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); new_connection_->socket_in_->connect(endpoint); new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); + new_connection_->socket_out_->connect( + "inproc://__inproc_in_endpoint__"); - // Forward on any messages sent to this channel before it existed. + // Forward on any messages sent to this channel before it existed. vector::iterator i; for (i = undelivered_messages_[*new_connection_channel_].begin(); i != undelivered_messages_[*new_connection_channel_].end(); ++i) { @@ -183,13 +194,13 @@ void ConnectionMultiplexer::Run() { undelivered_messages_.erase(*new_connection_channel_); } - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); + if ((new_connection_channel_->substr(0, 9) == "scheduler") && + (new_connection_channel_->substr(9, 1) != "_")) { + link_unlink_queue_[*new_connection_channel_] = + new AtomicQueue(); } // Reset request variable. new_connection_channel_ = NULL; - } // Serve any pending (valid) connection deletion request. @@ -213,43 +224,41 @@ void ConnectionMultiplexer::Run() { // local Link/UnlinkChannel requests. if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); + // Normal message. Forward appropriately. + Send(message); } - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - + for (unordered_map*>::iterator it = + link_unlink_queue_.begin(); + it != link_unlink_queue_.end(); ++it) { + MessageProto message; + bool got_it = it->second->Pop(&message); + if (got_it == true) { + if (message.type() == MessageProto::LINK_CHANNEL) { + remote_result_[message.channel_request()] = remote_result_[it->first]; + // Forward on any messages sent to this channel before it existed. + vector::iterator i; + for (i = undelivered_messages_[message.channel_request()].begin(); + i != undelivered_messages_[message.channel_request()].end(); + ++i) { + Send(*i); + } + undelivered_messages_.erase(message.channel_request()); + } else if (message.type() == MessageProto::UNLINK_CHANNEL) { + remote_result_.erase(message.channel_request()); + } + } + } } } // Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { +void* ConnectionMultiplexer::RunMultiplexer(void* multiplexer) { reinterpret_cast(multiplexer)->Run(); return NULL; } void ConnectionMultiplexer::Send(const MessageProto& message) { - if (message.type() == MessageProto::READ_RESULT) { if (remote_result_.count(message.destination_channel()) > 0) { remote_result_[message.destination_channel()]->Push(message); @@ -257,20 +266,18 @@ void ConnectionMultiplexer::Send(const MessageProto& message) { undelivered_messages_[message.destination_channel()].push_back(message); } } else { - // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); // Send message. if (message.destination_node() == configuration_->this_node_id) { // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. + // message on, else store it to be delivered if the channel is ever + // created. if (inproc_out_.count(message.destination_channel()) > 0) inproc_out_[message.destination_channel()]->send(msg); else @@ -278,10 +285,10 @@ void ConnectionMultiplexer::Send(const MessageProto& message) { } else { // Message is addressed to valid remote node. Channel validity will be // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); + pthread_mutex_lock(&send_mutex_[message.destination_node()]); remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } + pthread_mutex_unlock(&send_mutex_[message.destination_node()]); + } } } @@ -304,7 +311,8 @@ Connection::~Connection() { // Wait for the Run() loop to delete its socket for this Connection object. // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} + while (multiplexer_->delete_connection_channel_ != NULL) { + } // Allow future calls to NewConnection/~Connection. pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); @@ -314,11 +322,9 @@ void Connection::Send(const MessageProto& message) { // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); // Send message. socket_out_->send(msg); } @@ -327,12 +333,10 @@ void Connection::Send1(const MessageProto& message) { // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); + pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); multiplexer()->remote_out_[message.destination_node()]->send(msg); pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); } @@ -376,4 +380,3 @@ void Connection::UnlinkChannel(const string& channel) { m.set_channel_request(channel); multiplexer()->link_unlink_queue_[channel_]->Push(m); } - diff --git a/src_calvin/common/connection.h b/src_calvin/common/connection.h index 5fe5f673..47072b28 100644 --- a/src_calvin/common/connection.h +++ b/src_calvin/common/connection.h @@ -16,7 +16,6 @@ #include #include - #include "common/zmq.hpp" #include "proto/message.pb.h" #include "common/utils.h" @@ -47,8 +46,9 @@ class ConnectionMultiplexer { // the channel name is already in use, in which case NULL is returned. The // caller (not the multiplexer) owns of the newly created Connection object. Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); + + Connection* NewConnection(const string& channel, + AtomicQueue** aa); zmq::context_t* context() { return &context_; } @@ -60,7 +60,7 @@ class ConnectionMultiplexer { void Run(); // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); + static void* RunMultiplexer(void* multiplexer); // TODO(alex): Comments. void Send(const MessageProto& message); @@ -92,9 +92,9 @@ class ConnectionMultiplexer { // Sockets for forwarding messages to Connections. Keyed by channel // name. Type = ZMQ_PUSH. unordered_map inproc_out_; - + unordered_map*> remote_result_; - + unordered_map*> link_unlink_queue_; // Stores messages addressed to local channels that do not exist at the time @@ -106,9 +106,9 @@ class ConnectionMultiplexer { // Protects concurrent calls to NewConnection(). pthread_mutex_t new_connection_mutex_; - + pthread_mutex_t* send_mutex_; - + // Specifies a requested channel. Null if there is no outstanding new // connection request. const string* new_connection_channel_; @@ -137,7 +137,7 @@ class Connection { // Sends 'message' to the Connection specified by // 'message.destination_node()' and 'message.destination_channel()'. void Send(const MessageProto& message); - + void Send1(const MessageProto& message); // Loads the next incoming MessageProto into 'message'. Returns true, unless @@ -194,4 +194,3 @@ class Connection { }; #endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_calvin/common/debug.hh b/src_calvin/common/debug.hh new file mode 100644 index 00000000..c20fe396 --- /dev/null +++ b/src_calvin/common/debug.hh @@ -0,0 +1,9 @@ +#pragma once + +#include +#include +#include // これを追加 + +static void PrintCpu(std::string worker_name, int id) { + std::cout << worker_name << " is on cpu " << sched_getcpu() << std::endl; +} \ No newline at end of file diff --git a/src_calvin/common/definitions.hh b/src_calvin/common/definitions.hh new file mode 100644 index 00000000..22a9272b --- /dev/null +++ b/src_calvin/common/definitions.hh @@ -0,0 +1,11 @@ +#pragma once + +#define MAX_ACTIVE_TXNS 2000 // default 2000 +#define LOCK_BATCH_SIZE 100 // default 100 + +#define HOT 10 + +#define COLD_CUTOFF 990000 // default 990000 + +#define RW_SET_SIZE 1 // MUST BE EVEN, default 10 +#define DB_SIZE 1000000 // default 1000000 \ No newline at end of file diff --git a/src_calvin/common/inline.hh b/src_calvin/common/inline.hh new file mode 100644 index 00000000..936b3b69 --- /dev/null +++ b/src_calvin/common/inline.hh @@ -0,0 +1,3 @@ +#pragma once + +#define INLINE __attribute__((always_inline)) inline diff --git a/src_calvin/common/random.hh b/src_calvin/common/random.hh new file mode 100644 index 00000000..350c10c5 --- /dev/null +++ b/src_calvin/common/random.hh @@ -0,0 +1,132 @@ +/** + * This algorithm is originally developed + * by David Blackman and Sebastiano Vigna (vigna@acm.org) + * http://xoroshiro.di.unimi.it/xoroshiro128plus.c + * + * And Tanabe Takayuki custmized. + * + * Additional methods for specific distributions (Uniform, Zipf) were integrated. + */ + +#pragma once + +#include +#include +#include + +class Xoroshiro128Plus +{ +public: + Xoroshiro128Plus(uint64_t seed = 0) + { + if (seed == 0) + { + std::random_device rd; + s[0] = (static_cast(rd()) << 32) | rd(); + s[1] = (static_cast(rd()) << 32) | rd(); + } + else + { + s[0] = splitMix64(seed); + s[1] = splitMix64(s[0]); + } + } + + uint64_t s[2]; + + // Generates the next 64-bit pseudo-random number. + uint64_t next(void) + { + const uint64_t s0 = s[0]; + uint64_t s1 = s[1]; + const uint64_t result = s0 + s1; + + s1 ^= s0; + s[0] = rotl(s0, 24) ^ s1 ^ (s1 << 16); // a, b + s[1] = rotl(s1, 37); // c + + return result; + } + + // Overload operator() to be used as a standard generator. + uint64_t operator()() { return next(); } + + // ------------------- ここから追加するメソッド ------------------- + + // Generates a random double in [0.0, 1.0). + double NextUniform() + { + return (next() >> 11) * (1.0 / (1LL << 53)); + } + + // Generates a random integer uniformly distributed in [0, n-1]. + uint64_t Uniform(uint64_t n) + { + return next() % n; + } + + // Generates a random integer from a Zipfian distribution in [0, n-1]. + uint64_t Zipf(uint64_t n, double theta) + { + if (!zipf_generator_initialized || n != zipf_n || theta != zipf_theta) + { + init_zipf_generator(n, theta); + } + + double alpha = 1.0 / (1.0 - theta); + double zetan = zipf_zeta_n; + double eta = (1.0 - std::pow(2.0 / n, 1.0 - theta)) / + (1.0 - zipf_zeta_2 / zetan); + + double u = NextUniform(); + double uz = u * zetan; + + if (uz < 1.0) + return 0; + if (uz < 1.0 + std::pow(0.5, theta)) + return 1; + + return static_cast(n * std::pow(eta * u - eta + 1.0, alpha)); + } + + // ------------------- 追加メソッドここまで ------------------- + +private: + // For seeding the generator. + uint64_t splitMix64(uint64_t seed) + { + uint64_t z = (seed += 0x9e3779b97f4a7c15); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) * 0x94d049bb133111eb; + return z ^ (z >> 31); + } + + inline uint64_t rotl(const uint64_t x, int k) + { + return (x << k) | (x >> (64 - k)); + } + + // --- Zipfian distribution helper variables and functions --- + // mutable allows these to be modified even in const methods. + mutable bool zipf_generator_initialized = false; + mutable uint64_t zipf_n = 0; + mutable double zipf_theta = 0.0; + mutable double zipf_zeta_n = 0.0; + mutable double zipf_zeta_2 = 0.0; + + // Pre-computes constants for the Zipfian distribution. + void init_zipf_generator(uint64_t n, double theta) const + { + Xoroshiro128Plus *non_const_this = const_cast(this); + non_const_this->zipf_zeta_n = 0; + for (uint64_t i = 0; i < n; i++) + { + non_const_this->zipf_zeta_n += 1 / std::pow(i + 1, theta); + } + non_const_this->zipf_zeta_2 = 1 + (1 / std::pow(2, theta)); + non_const_this->zipf_n = n; + non_const_this->zipf_theta = theta; + non_const_this->zipf_generator_initialized = true; + } +}; +using Random = Xoroshiro128Plus; \ No newline at end of file diff --git a/src_calvin/common/test_template.cc b/src_calvin/common/test_template.cc index 9dc11631..4f4175d3 100644 --- a/src_calvin/common/test_template.cc +++ b/src_calvin/common/test_template.cc @@ -11,4 +11,3 @@ TEST(MyTest) { int main(int argc, char** argv) { MyTest(); } - diff --git a/src_calvin/common/testing.h b/src_calvin/common/testing.h index 83e27c03..d56ba735 100644 --- a/src_calvin/common/testing.h +++ b/src_calvin/common/testing.h @@ -37,7 +37,7 @@ using namespace std; // Don't do this at home, kids. bool __failed_; #define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ +#define CHECK(T, MSG) \ do { \ if (!(T)) { \ __failed_ = true; \ @@ -45,52 +45,54 @@ bool __failed_; } \ } while (0) -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " +#define LINE cout << "[ " << __FUNCTION__ << " ] " -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ +#define EXPECT_TRUE(T) \ + do { \ + if (!(T)) { \ + __failed_ = true; \ + cout << "EXPECT_TRUE(" << #T << ") failed at " << __FILE__ << ":" \ + << __LINE__ << "\n"; \ + } \ } while (0) -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ +#define EXPECT_FALSE(T) \ + do { \ + if (T) { \ + __failed_ = true; \ + cout << "EXPECT_FALSE(" << #T << ") failed at " << __FILE__ << ":" \ + << __LINE__ << "\n"; \ + } \ } while (0) -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ +#define EXPECT_EQ(A, B) \ + do { \ + if ((A) != (B)) { \ + __failed_ = true; \ + cout << "EXPECT_EQ(" << #A << ", " << #B \ + << ") \033[1;31mfailed\033[0m at " << __FILE__ << ":" << __LINE__ \ + << "\n" \ + << "Expected:\n" \ + << A << "\n" \ + << "Actual:\n" \ + << B << "\n"; \ + } \ } while (0) -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); +#define TEST(TESTNAME) \ + void TESTNAME() { \ + __failed_ = false; \ + LINE << "\033[1;32mBEGIN\033[0m\n"; \ + do +#define END \ + if (__failed_) { \ + LINE << "\033[1;31mFAIL\033[0m\n"; \ + } else { \ + LINE << "\033[1;32mPASS\033[0m\n"; \ + } \ + } \ + while (0) \ + ; #endif // _DB_COMMON_TESTING_H_ diff --git a/src_calvin/common/types.h b/src_calvin/common/types.h index 6778d855..c6cb7dde 100644 --- a/src_calvin/common/types.h +++ b/src_calvin/common/types.h @@ -28,24 +28,56 @@ typedef uint64_t uint64; typedef string bytes; // Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } +static inline bytes PackInt8(int8 x) { + return bytes((const char*)&x, 1); +} +static inline bytes PackInt16(int16 x) { + return bytes((const char*)&x, 2); +} +static inline bytes PackInt32(int32 x) { + return bytes((const char*)&x, 4); +} +static inline bytes PackInt64(int64 x) { + return bytes((const char*)&x, 8); +} +static inline int8 UnpackInt8(bytes s) { + return *((int8*)(s.data())); +} +static inline int16 UnpackInt16(bytes s) { + return *((int16*)(s.data())); +} +static inline int32 UnpackInt32(bytes s) { + return *((int32*)(s.data())); +} +static inline int64 UnpackInt64(bytes s) { + return *((int64*)(s.data())); +} // Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } +static inline bytes PackUInt8(uint8 x) { + return bytes((const char*)&x, 1); +} +static inline bytes PackUInt16(uint16 x) { + return bytes((const char*)&x, 2); +} +static inline bytes PackUInt32(uint32 x) { + return bytes((const char*)&x, 4); +} +static inline bytes PackUInt64(uint64 x) { + return bytes((const char*)&x, 8); +} +static inline uint8 UnpackUInt8(bytes s) { + return *((uint8*)(s.data())); +} +static inline uint16 UnpackUInt16(bytes s) { + return *((uint16*)(s.data())); +} +static inline uint32 UnpackUInt32(bytes s) { + return *((uint32*)(s.data())); +} +static inline uint64 UnpackUInt64(bytes s) { + return *((uint64*)(s.data())); +} // Key type for database objects. // Note: if this changes from bytes, the types need to be updated for the diff --git a/src_calvin/common/utils.h b/src_calvin/common/utils.h index ea1021cb..012a83a4 100644 --- a/src_calvin/common/utils.h +++ b/src_calvin/common/utils.h @@ -24,7 +24,11 @@ using std::tr1::unordered_map; #define ASSERTS_ON true -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) +#define DCHECK(ARG) \ + do { \ + if (ASSERTS_ON) \ + assert(ARG); \ + } while (0) // Status code for return values. struct Status { @@ -50,9 +54,12 @@ struct Status { // Pretty printing. string ToString() { string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); + if (code == ERROR) + out.append("Error"); + if (code == OKAY) + out.append("Okay"); + if (code == DONE) + out.append("Done"); if (message.size()) { out.append(": "); out.append(message); @@ -66,19 +73,20 @@ struct Status { static inline double GetTime() { struct timeval tv; gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; + return tv.tv_sec + tv.tv_usec / 1e6; } // Busy-wait for 'duration' seconds. static inline void Spin(double duration) { usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} + // double start = GetTime(); + // while (GetTime() < start + duration) {} } // Busy-wait until GetTime() >= time. static inline void SpinUntil(double time) { - while (GetTime() >= time) {} + while (GetTime() >= time) { + } } // Produces a random alphabet string of the specified length @@ -113,8 +121,8 @@ static inline double StringToDouble(const string& s) { } static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); + double f = (double)rand() / RAND_MAX; + return fMin + f * (fMax - fMin); } // Converts a human-readable numeric sub-string (starting at the 'n'th position @@ -139,9 +147,7 @@ static inline void Noop(void* data, void* hint) {} class Mutex { public: // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } + Mutex() { pthread_mutex_init(&mutex_, NULL); } private: friend class Lock; @@ -158,9 +164,7 @@ class Lock { explicit Lock(Mutex* mutex) : mutex_(mutex) { pthread_mutex_lock(&mutex_->mutex_); } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } + ~Lock() { pthread_mutex_unlock(&mutex_->mutex_); } private: Mutex* mutex_; @@ -175,7 +179,7 @@ class Lock { //////////////////////////////////////////////////////////////// -template +template class AtomicQueue { public: AtomicQueue() { @@ -192,21 +196,19 @@ class AtomicQueue { } // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } + inline bool Empty() { return front_ == back_; } // Atomically pushes 'item' onto the queue. inline void Push(const T& item) { Lock l(&back_mutex_); // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { + if (front_ == (back_ + 1) % size_) { Lock m(&front_mutex_); Lock n(&size_mutex_); uint32 count = (back_ + size_ - front_) % size_; queue_.resize(size_ * 2); for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; + queue_[size_ + i] = queue_[(front_ + i) % size_]; } front_ = size_; back_ = size_ + count; @@ -260,9 +262,7 @@ class AtomicQueue { class MutexRW { public: // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } + MutexRW() { pthread_rwlock_init(&mutex_, NULL); } private: friend class ReadLock; @@ -280,9 +280,7 @@ class ReadLock { explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { pthread_rwlock_rdlock(&mutex_->mutex_); } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } + ~ReadLock() { pthread_rwlock_unlock(&mutex_->mutex_); } private: MutexRW* mutex_; @@ -300,9 +298,7 @@ class WriteLock { explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { pthread_rwlock_wrlock(&mutex_->mutex_); } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } + ~WriteLock() { pthread_rwlock_unlock(&mutex_->mutex_); } private: MutexRW* mutex_; @@ -315,7 +311,7 @@ class WriteLock { WriteLock& operator=(const WriteLock&); }; -template +template class AtomicMap { public: AtomicMap() {} @@ -362,7 +358,7 @@ class AtomicMap { inline void DeleteVAndClear() { WriteLock l(&mutex_); for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { + it != map_.end(); ++it) { delete it->second; } map_.clear(); @@ -378,4 +374,3 @@ class AtomicMap { }; #endif // _DB_COMMON_UTILS_H_ - diff --git a/src_calvin/common/zipf.hh b/src_calvin/common/zipf.hh new file mode 100644 index 00000000..93f24bb5 --- /dev/null +++ b/src_calvin/common/zipf.hh @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "inline.hh" +#include "random.hh" + +using std::cout; +using std::endl; + +// Fast zipf distribution by Jim Gray et al. +class FastZipf { + Xoroshiro128Plus* rnd_; + const size_t nr_; + const double alpha_, zetan_, eta_; + const double threshold_; + + public: + FastZipf(Xoroshiro128Plus* rnd, double theta, size_t nr) + : rnd_(rnd), + nr_(nr), + alpha_(1.0 / (1.0 - theta)), + zetan_(zeta(nr, theta)), + eta_((1.0 - std::pow(2.0 / (double)nr, 1.0 - theta)) / + (1.0 - zeta(2, theta) / zetan_)), + threshold_(1.0 + std::pow(0.5, theta)) { + assert(0.0 <= theta); + assert(theta < 1.0); // 1.0 can not be specified. + } + + // Use this constructor if zeta is pre-calculated. + FastZipf(Xoroshiro128Plus* rnd, double theta, size_t nr, double zetan) + : rnd_(rnd), + nr_(nr), + alpha_(1.0 / (1.0 - theta)), + zetan_(zetan), + eta_((1.0 - std::pow(2.0 / (double)nr, 1.0 - theta)) / + (1.0 - zeta(2, theta) / zetan_)), + threshold_(1.0 + std::pow(0.5, theta)) { + assert(0.0 <= theta); + assert(theta < 1.0); // 1.0 can not be specified. + } + + INLINE size_t operator()() { + double u = rnd_->next() / (double)UINT64_MAX; + double uz = u * zetan_; + if (uz < 1.0) + return 0; + if (uz < threshold_) + return 1; + return (size_t)((double)nr_ * std::pow(eta_ * u - eta_ + 1.0, alpha_)); + } + + uint64_t rand() { return rnd_->next(); } + + static double zeta(size_t nr, double theta) { + double ans = 0.0; + for (size_t i = 0; i < nr; ++i) + ans += std::pow(1.0 / (double)(i + 1), theta); + return ans; + } +}; diff --git a/src_calvin/common/zmq.hpp b/src_calvin/common/zmq.hpp index 56653329..9fed64c9 100644 --- a/src_calvin/common/zmq.hpp +++ b/src_calvin/common/zmq.hpp @@ -28,234 +28,166 @@ #include #include -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; +namespace zmq { +typedef zmq_free_fn free_fn; +typedef zmq_pollitem_t pollitem_t; + +class error_t : public std::exception { + public: + error_t() : errnum(zmq_errno()) {} + + virtual const char* what() const throw() { return zmq_strerror(errnum); } + + int num() const { return errnum; } + + private: + int errnum; +}; + +inline int poll(zmq_pollitem_t* items_, int nitems_, long timeout_ = -1) { + return zmq_poll(items_, nitems_, timeout_); +} + +inline void device(int device_, void* insocket_, void* outsocket_) { + zmq_device(device_, insocket_, outsocket_); } +inline void version(int* major_, int* minor_, int* patch_) { + zmq_version(major_, minor_, patch_); +} + +class message_t : private zmq_msg_t { + friend class socket_t; + + public: + inline message_t() { zmq_msg_init(this); } + + inline message_t(size_t size_) { zmq_msg_init_size(this, size_); } + + inline message_t(void* data_, + size_t size_, + free_fn* ffn_, + void* hint_ = NULL) { + zmq_msg_init_data(this, data_, size_, ffn_, hint_); + } + + inline ~message_t() { + int rc = zmq_msg_close(this); + assert(rc == 0); + } + + inline void rebuild() { + zmq_msg_close(this); + zmq_msg_init(this); + } + + inline void rebuild(size_t size_) { + zmq_msg_close(this); + zmq_msg_init_size(this, size_); + } + + inline void rebuild(void* data_, + size_t size_, + free_fn* ffn_, + void* hint_ = NULL) { + zmq_msg_close(this); + zmq_msg_init_data(this, data_, size_, ffn_, hint_); + } + + inline void move(message_t* msg_) { zmq_msg_move(this, (zmq_msg_t*)msg_); } + + inline void copy(message_t* msg_) { zmq_msg_copy(this, (zmq_msg_t*)msg_); } + + inline void* data() { return zmq_msg_data(this); } + + inline size_t size() { return zmq_msg_size(this); } + + private: + // Disable implicit message copying, so that users won't use shared + // messages (less efficient) without being aware of the fact. + message_t(const message_t&); + void operator=(const message_t&); +}; + +class context_t { + friend class socket_t; + + public: + inline context_t(int io_threads_) { ptr = zmq_init(io_threads_); } + + inline ~context_t() { + int rc = zmq_term(ptr); + assert(rc == 0); + } + + // Be careful with this, it's probably only useful for + // using the C api together with an existing C++ api. + // Normally you should never need to use this. + inline operator void*() { return ptr; } + + private: + void* ptr; + + context_t(const context_t&); + void operator=(const context_t&); +}; + +class socket_t { + public: + inline socket_t(context_t& context_, int type_) { + ptr = zmq_socket(context_.ptr, type_); + } + + inline ~socket_t() { close(); } + + inline operator void*() { return ptr; } + + inline void close() { + if (ptr == NULL) + // already closed + return; + zmq_close(ptr); + ptr = 0; + } + + inline void setsockopt(int option_, const void* optval_, size_t optvallen_) { + zmq_setsockopt(ptr, option_, optval_, optvallen_); + } + + inline void getsockopt(int option_, void* optval_, size_t* optvallen_) { + zmq_getsockopt(ptr, option_, optval_, optvallen_); + } + + inline void bind(const char* addr_) { zmq_bind(ptr, addr_); } + + inline void connect(const char* addr_) { zmq_connect(ptr, addr_); } + + inline bool send(message_t& msg_, int flags_ = 0) { + int rc = zmq_send(ptr, &msg_, flags_); + if (rc == 0) + return true; + if (rc == -1 && zmq_errno() == EAGAIN) + return false; + return false; + } + + inline bool recv(message_t* msg_, int flags_ = 0) { + int rc = zmq_recv(ptr, msg_, flags_); + if (rc == 0) + return true; + if (rc == -1 && zmq_errno() == EAGAIN) + return false; + return false; + } + + private: + void* ptr; + + socket_t(const socket_t&); + void operator=(const socket_t&); +}; + +} // namespace zmq + #endif diff --git a/src_calvin/deployment/addhosts.pl b/src_calvin/deployment/addhosts.pl old mode 100755 new mode 100644 diff --git a/src_calvin/deployment/cluster.cc b/src_calvin/deployment/cluster.cc index c04f82b9..9f82aa1e 100644 --- a/src_calvin/deployment/cluster.cc +++ b/src_calvin/deployment/cluster.cc @@ -30,8 +30,8 @@ int UpdatePorts(int port_begin, Configuration* config); void Deploy(const Configuration& config, const char* exec); const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; +const char default_port_filename[] = "deployment/portfile"; +const char default_executable_filename[] = "../obj/deployment/db"; const char default_run_config_filename[] = "deploy-run.conf"; const char remote_exec[] = "ssh"; @@ -48,7 +48,6 @@ const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; // sprintf(remote_opt3, remote_opt3_fmt, // cwd, per-exec, node-id, db-args (joined with spaces)) - // TODO(scw): make deployer class; these should be class objs char* cwd; char* db_args; @@ -66,20 +65,19 @@ int main(int argc, char* argv[]) { int arg_begin; const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; + const char* port_file = default_port_filename; + const char* exec = default_executable_filename; for (arg_begin = 1; arg_begin < argc; ++arg_begin) { if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); + printf( + "Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" + " [-d db-exec] [db-args..]\n" + " -c config-file default: %s\n" + " -p port-file default: %s\n" + " -d db-exec default: %s\n", + argv[0], default_input_config_filename, default_port_filename, + default_executable_filename); return 0; } else if (strcmp(argv[arg_begin], "-c") == 0) { config_file = argv[++arg_begin]; @@ -133,14 +131,14 @@ int main(int argc, char* argv[]) { fprintf(port_fp, "%d\n", next_port); fclose(port_fp); -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } + // for (map::iterator it = config.all_nodes.begin(); + // it != config.all_nodes.end(); ++it) { + // char copy_config[1024]; + // snprintf(copy_config, sizeof(copy_config), + // "scp -rp deploy-run.conf %s:db3/deploy-run.conf", + // it->second->host.c_str()); + // system(copy_config); + // } Deploy(config, exec); @@ -181,8 +179,7 @@ int UpdatePorts(int port_begin, Configuration* config) { // Insert host, port_begin> if the host has not appeared. // Otherwise, use the existing host-port pair. map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; + next_port_map.insert(std::make_pair(node->host, port_begin)).first; node->port = port_it->second; port_it->second = port_it->second + 1; @@ -212,14 +209,14 @@ void DeployOne(int nodeID, char remote_opt3[1024]; if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, cwd, + exec, nodeID, db_args); else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, cwd, exec, + nodeID, db_args); else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, cwd, exec, + nodeID, db_args); // Black magic, don't touch (bug scw if this breaks). int pipefd[2]; @@ -242,10 +239,10 @@ void DeployOne(int nodeID, close(pipefd[1]); } - timespec to_sleep = { 0, 100000000 }; // 0.1 sec + timespec to_sleep = {0, 100000000}; // 0.1 sec nanosleep(&to_sleep, NULL); - (void) config_file; + (void)config_file; } void TerminatingChildren(int sig); @@ -334,7 +331,7 @@ void Deploy(const Configuration& config, const char* exec) { // kill all active components & all ssh procs KillRemote(config, exec, false); - timespec to_sleep = { 1, 0 }; // 1 sec + timespec to_sleep = {1, 0}; // 1 sec nanosleep(&to_sleep, NULL); KillLocal(); } @@ -346,7 +343,8 @@ void TerminatingChildren(int sig) { // try to kill all remote processes spawned void KillRemote(const Configuration& config, - const char* exec, bool client_int) { + const char* exec, + bool client_int) { const char* sig_arg; if (client_int) sig_arg = "-INT"; @@ -354,17 +352,16 @@ void KillRemote(const Configuration& config, sig_arg = "-TERM"; char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); + snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", cwd, exec); - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { + for (map::const_iterator it = config.all_nodes.begin(); + it != config.all_nodes.end(); ++it) { Node* node = it->second; int pid = fork(); if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); + execlp("ssh", "ssh", node->host.c_str(), "killall", sig_arg, + exec_fullpath, NULL); exit(-1); } } diff --git a/src_calvin/deployment/main.cc b/src_calvin/deployment/main.cc index 89b8f32e..d4d976ec 100644 --- a/src_calvin/deployment/main.cc +++ b/src_calvin/deployment/main.cc @@ -6,11 +6,15 @@ #include #include #include +#include +#include "applications/application.h" #include "applications/microbenchmark.h" #include "applications/tpcc.h" +#include "applications/ycsb.h" #include "common/configuration.h" #include "common/connection.h" +#include "common/definitions.hh" #include "backend/simple_storage.h" #include "backend/fetching_storage.h" #include "backend/collapsed_versioned_storage.h" @@ -18,9 +22,9 @@ #include "scheduler/deterministic_scheduler.h" #include "sequencer/sequencer.h" #include "proto/tpcc_args.pb.h" +#include "proto/txn.pb.h" -#define HOT 10000 - +// グローバル変数 map latest_order_id_for_customer; map latest_order_id_for_district; map smallest_order_id_for_district; @@ -28,153 +32,218 @@ map customer_for_order; unordered_map next_order_id_for_district; map item_for_order_line; map order_line_number; - -vector* involed_customers; +vector *involed_customers; pthread_mutex_t mutex_; pthread_mutex_t mutex_for_item; -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other; - do { - other = rand() % config_->all_nodes.size(); - } while (other == config_->this_node_id); - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other); - } else { - // Single-partition txn. - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); +// Microbenchmark用クライアント +class MClient : public Client +{ +public: + // --- MODIFICATION START --- + // コンストラクタの第3引数の型から const を外す + MClient(Configuration *config, int mp, Application *app) + // microbenchmark_ を const ではないポインタとして初期化 + : microbenchmark_(static_cast(app)), config_(config), + percent_mp_(mp) + { + } + // --- MODIFICATION END --- + + virtual ~MClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + if (config_->all_nodes.size() > 1 && (rand() % 100) < percent_mp_) + { + int other; + do + { + other = rand() % config_->all_nodes.size(); + } while (other == config_->this_node_id); + // これでエラーなく呼び出せる + *txn = microbenchmark_->MicroTxnMP(txn_id, config_->this_node_id, other); + } + else + { + *txn = microbenchmark_->MicroTxnSP(txn_id, config_->this_node_id); + } } - } - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; +private: + // --- MODIFICATION START --- + // メンバ変数の型から const を外す + Microbenchmark *microbenchmark_; + // --- MODIFICATION END --- + Configuration *config_; + int percent_mp_; }; -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); +// TPCC用クライアント +class TClient : public Client +{ +public: + TClient(Configuration *config, int mp, Application *app) + : config_(config), percent_mp_(mp), tpcc_(static_cast(app)) {} + virtual ~TClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + TPCCArgs args; + args.set_system_time(GetTime()); + args.set_multipartition((rand() % 100) < percent_mp_); + + string args_string; + args.SerializeToString(&args_string); + + int random_txn_type = rand() % 100; + if (random_txn_type < 45) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); + } + else if (random_txn_type < 88) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); + } + else if (random_txn_type < 92) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); + } + else if (random_txn_type < 96) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); + } + else + { + *txn = tpcc_->NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); + } } - } +private: + Configuration *config_; + int percent_mp_; + TPCC *tpcc_; +}; - private: - Configuration* config_; - int percent_mp_; +// YCSB用クライアント +class YClient : public Client +{ +public: + YClient(Configuration *config, int mp, Application *ycsb_app) + : config_(config), percent_mp_(mp), ycsb_app_(ycsb_app) {} + virtual ~YClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + string args_string = ""; + *txn = ycsb_app_->NewTxn(txn_id, 0, args_string, config_); + } + +private: + Configuration *config_; + int percent_mp_; + Application *ycsb_app_; }; -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); +// シグナルハンドラ +void stop(int sig) +{ + exit(sig); } -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, - storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { +int main(int argc, char **argv) +{ + // 引数チェック + if (argc < 4) + { + fprintf(stderr, "Usage: %s [f for fetching]\n", + argv[0]); + exit(1); + } + bool useFetching = (argc > 4 && argv[4][0] == 'f'); + + signal(SIGINT, &stop); + signal(SIGTERM, &stop); + + // 設定オブジェクトの構築 + Configuration config(StringToInt(argv[1]), "deploy-run.conf"); + + // 通信マルチプレクサの構築と起動 + ConnectionMultiplexer multiplexer(&config); + + // ベンチマークの種類に応じてApplicationオブジェクトを生成 + Application *application; + if (argv[2][0] == 'm') + { + application = new Microbenchmark(config.all_nodes.size(), HOT); + } + else if (argv[2][0] == 't') + { + application = new TPCC(); + } + else if (argv[2][0] == 'y') + { + // YCSBのパラメータをここで自由に設定 + const double read_ratio = 1.0; + const double skew = 0.99; + const uint64 db_size = DB_SIZE; + const uint64 hot_records = 10; // この値はdefinitions.hhに移動するとより良い + application = new YCSB(read_ratio, skew, db_size, hot_records); + } + else + { + fprintf(stderr, "Unknown benchmark type: %s\n", argv[2]); + exit(1); + } + + // Applicationオブジェクトに対応するClientオブジェクトを生成 + Client *client; + if (argv[2][0] == 'm') + { + client = new MClient(&config, atoi(argv[3]), application); + } + else if (argv[2][0] == 't') + { + client = new TClient(&config, atoi(argv[3]), application); + } + else // 'y' + { + client = new YClient(&config, atoi(argv[3]), application); + } + + pthread_mutex_init(&mutex_, NULL); + pthread_mutex_init(&mutex_for_item, NULL); + involed_customers = new vector; + + // ストレージ層の選択と初期化 + Storage *storage; + if (!useFetching) + { + storage = new SimpleStorage(); + } + else + { + storage = FetchingStorage::BuildStorage(); + } + storage->Initmutex(); + application->InitializeStorage(storage, &config); + + // シーケンサコンポーネントの初期化と起動 + Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, + storage); + + // スケジューラをメインスレッドで実行 DeterministicScheduler scheduler(&config, multiplexer.NewConnection("scheduler_"), storage, - new TPCC()); - } + application); - Spin(180); - return 0; -} + // 180秒間実行 + Spin(180); + + // メモリ解放 + delete client; + delete application; + delete storage; + delete involed_customers; + return 0; +} \ No newline at end of file diff --git a/src_calvin/deployment/makeconf.pl b/src_calvin/deployment/makeconf.pl old mode 100755 new mode 100644 diff --git a/src_calvin/obj/.deps b/src_calvin/obj/.deps deleted file mode 100644 index e69de29b..00000000 diff --git a/src_calvin/obj/common/configuration.d b/src_calvin/obj/common/configuration.d deleted file mode 100644 index d4246650..00000000 --- a/src_calvin/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_calvin/paxos/paxos.cc b/src_calvin/paxos/paxos.cc index d0523e10..17e7d624 100644 --- a/src_calvin/paxos/paxos.cc +++ b/src_calvin/paxos/paxos.cc @@ -21,15 +21,15 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { if (s.substr(0, 10) == "clientPort") { int pos1 = s.find('='); int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); + port = s.substr(pos1 + 1, pos2 - pos1 - 1); } else if (s.substr(0, 7) == "server1") { int pos1 = s.find('='); int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); + ip = s.substr(pos1 + 1, pos2 - pos1 - 1); } else if (s.substr(0, 7) == "timeout") { int pos1 = s.find('='); int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); + timeout = s.substr(pos1 + 1, pos2 - pos1 - 1); } } connection_string = ip + ":" + port; @@ -42,8 +42,8 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { } // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); + zh_ = zookeeper_init(connection_string.c_str(), NULL, atoi(timeout.c_str()), + 0, NULL, 0); if (zh_ == NULL) { printf("Connection to zookeeper failed.\n"); return; @@ -56,9 +56,8 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { // If multiple nodes executing this code to both see that // the root doesn't exist, only one node creates /root // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); + int create_rc = + zoo_create(zh_, "/root", NULL, 0, &ZOO_OPEN_ACL_UNSAFE, 0, NULL, 0); if (create_rc != ZOK && create_rc != ZNODEEXISTS) { printf("zoo_create error:error number is %d\n", create_rc); } @@ -70,10 +69,9 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { char current_read_batch_path[23]; snprintf(current_read_batch_path, sizeof(current_read_batch_path), "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); + int get_rc = zoo_aget( + zh_, current_read_batch_path, 0, get_data_completion, + reinterpret_cast(new pair(i, this))); if (get_rc) { printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); } @@ -97,8 +95,7 @@ void Paxos::SubmitBatch(const string& batch_data) { // Submit batch means that create new znode below the root directory. int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); + ZOO_SEQUENCE | ZOO_EPHEMERAL, acreate_completion, NULL); if (rc != ZOK) { printf("zoo_acreate error:error number is %d\n", rc); } @@ -127,8 +124,11 @@ void Paxos::GetNextBatchBlocking(string* batch_data) { } } -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { +void Paxos::get_data_completion(int rc, + const char* value, + int value_len, + const struct Stat* stat, + const void* data) { // XXX(scw): using const_cast is disgusting pair* previous_data = reinterpret_cast*>(const_cast(data)); @@ -142,34 +142,34 @@ void Paxos::get_data_completion(int rc, const char *value, int value_len, // Set the number of batch which will be got from zookeeper next time // (just plus the CONCURRENT_GETS). next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); + pthread_mutex_lock( + &paxos->mutexes_[previous_index_for_aget % CONCURRENT_GETS]); paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); + pthread_mutex_unlock( + &paxos->mutexes_[previous_index_for_aget % CONCURRENT_GETS]); // If there are no new batch in the zookeeper, just wait for a while // and continue to get from zookeeper. } else if (rc == ZNONODE) { next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); + usleep(0.2 * 1000); } else { return; } // Continue to get a batch from zookeeper. char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); + snprintf(current_read_batch_path, sizeof(current_read_batch_path), "%s%010lu", + "/root/batch-", next_index_for_aget); previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); + int get_rc = + zoo_aget(paxos->zh_, current_read_batch_path, 0, get_data_completion, + reinterpret_cast(previous_data)); if (get_rc) { return; } } -void Paxos::acreate_completion(int rc, const char *name, const void * data) { +void Paxos::acreate_completion(int rc, const char* name, const void* data) { if (rc) { printf("Error %d for zoo_acreate.\n", rc); } @@ -187,10 +187,10 @@ void StartZookeeper(const string& zookeeper_config_file) { if (line.substr(0, 6) == "server") { int pos1 = line.find('='); int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); + zookeepers.push_back(line.substr(pos1 + 1, pos2 - pos1 - 1)); } } - for (unsigned int i = 0; i< zookeepers.size(); i++) { + for (unsigned int i = 0; i < zookeepers.size(); i++) { // Generate the ssh command. string ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + @@ -205,8 +205,8 @@ void StartZookeeper(const string& zookeeper_config_file) { // This function will automatically stop zookeeper server based on the // zookeeper config file(generate ssh commands and execute them). void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; + vector zookeepers; + string line, port, ssh_command; // Read zookeeper config file. ifstream in(zookeeper_config_file.c_str()); // Put all zookeeper server's ip into the vector. @@ -214,24 +214,23 @@ void StopZookeeper(const string& zookeeper_config_file) { if (line.substr(0, 6) == "server") { int pos1 = line.find('='); int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); + zookeepers.push_back(line.substr(pos1 + 1, pos2 - pos1 - 1)); } if (line.substr(0, 10) == "clientPort") { int pos1 = line.find('='); int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); + port = line.substr(pos1 + 1, pos2 - pos1 - 1); } } ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; + " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " + + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; system(ssh_command.c_str()); sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { + for (unsigned int i = 0; i < zookeepers.size(); i++) { // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; + ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" + + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; system(ssh_command.c_str()); } } - diff --git a/src_calvin/paxos/paxos.h b/src_calvin/paxos/paxos.h index bbcd711d..f761ab25 100644 --- a/src_calvin/paxos/paxos.h +++ b/src_calvin/paxos/paxos.h @@ -53,7 +53,7 @@ class Paxos { private: // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; + zhandle_t* zh_; // Record the serial number of the batch which will be read next time. uint64 next_read_batch_index_; @@ -68,11 +68,14 @@ class Paxos { // For zoo_aget completion function, this method will be invoked // at the end of a asynchronous call( zoo_aget is asynchronous call // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); + static void get_data_completion(int rc, + const char* value, + int value_len, + const struct Stat* stat, + const void* data); // For zoo_acreate completion function, this method will be invoked // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); + static void acreate_completion(int rc, const char* name, const void* data); }; #endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_calvin/scheduler/deterministic_lock_manager.cc b/src_calvin/scheduler/deterministic_lock_manager.cc index 15d002e3..7741cee8 100644 --- a/src_calvin/scheduler/deterministic_lock_manager.cc +++ b/src_calvin/scheduler/deterministic_lock_manager.cc @@ -12,12 +12,10 @@ using std::vector; -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) +DeterministicLockManager::DeterministicLockManager(deque* ready_txns, + Configuration* config) + : configuration_(config), ready_txns_(ready_txns) { + for (int i = 0; i < LOCK_TABLE_SIZE; i++) lock_table_[i] = new deque(); } @@ -29,10 +27,11 @@ int DeterministicLockManager::Lock(TxnProto* txn) { // Only lock local keys. if (IsLocal(txn->read_write_set(i))) { deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - + deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { + for (it = key_requests->begin(); + it != key_requests->end() && it->key != txn->read_write_set(i); + ++it) { } deque* requests; if (it == key_requests->end()) { @@ -41,7 +40,7 @@ int DeterministicLockManager::Lock(TxnProto* txn) { } else { requests = it->locksrequest; } - + // Only need to request this if lock txn hasn't already requested it. if (requests->empty() || txn != requests->back().txn) { requests->push_back(LockRequest(WRITE, txn)); @@ -58,10 +57,10 @@ int DeterministicLockManager::Lock(TxnProto* txn) { // Only lock local keys. if (IsLocal(txn->read_set(i))) { deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - + deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { + for (it = key_requests->begin(); + it != key_requests->end() && it->key != txn->read_set(i); ++it) { } deque* requests; if (it == key_requests->end()) { @@ -70,7 +69,7 @@ int DeterministicLockManager::Lock(TxnProto* txn) { } else { requests = it->locksrequest; } - + // Only need to request this if lock txn hasn't already requested it. if (requests->empty() || txn != requests->back().txn) { requests->push_back(LockRequest(READ, txn)); @@ -100,9 +99,9 @@ void DeterministicLockManager::Release(TxnProto* txn) { Release(txn->read_set(i), txn); // Currently commented out because nothing in any write set can conflict // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); + // for (int i = 0; i < txn->write_set_size(); i++) + // if (IsLocal(txn->write_set(i))) + // Release(txn->write_set(i), txn); for (int i = 0; i < txn->read_write_set_size(); i++) if (IsLocal(txn->read_write_set(i))) Release(txn->read_write_set(i), txn); @@ -110,21 +109,19 @@ void DeterministicLockManager::Release(TxnProto* txn) { void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - + deque* key_requests = lock_table_[Hash(key)]; + deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { + for (it1 = key_requests->begin(); + it1 != key_requests->end() && it1->key != key; ++it1) { } deque* requests = it1->locksrequest; - // Seek to the target request. Note whether any write lock requests precede // the target. bool write_requests_precede_target = false; deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { + for (it = requests->begin(); it != requests->end() && it->txn != txn; ++it) { if (it->mode == WRITE) write_requests_precede_target = true; } @@ -154,8 +151,8 @@ void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { // If a sequence of read lock requests follows, grant all of them. for (; it != requests->end() && it->mode == READ; ++it) new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) + } else if (!write_requests_precede_target && target->mode == WRITE && + it->mode == READ) { // (c) // If a sequence of read lock requests follows, grant all of them. for (; it != requests->end() && it->mode == READ; ++it) new_owners.push_back(it->txn); @@ -177,9 +174,7 @@ void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { requests->erase(target); if (requests->size() == 0) { delete requests; - key_requests->erase(it1); + key_requests->erase(it1); } - } } - diff --git a/src_calvin/scheduler/deterministic_lock_manager.h b/src_calvin/scheduler/deterministic_lock_manager.h index 1bba5425..6416e90d 100644 --- a/src_calvin/scheduler/deterministic_lock_manager.h +++ b/src_calvin/scheduler/deterministic_lock_manager.h @@ -13,18 +13,18 @@ #include "common/configuration.h" #include "scheduler/lock_manager.h" #include "common/utils.h" +#include "common/definitions.hh" -using std::tr1::unordered_map; using std::deque; +using std::tr1::unordered_map; -#define TABLE_SIZE 1000000 +// #define LOCK_TABLE_SIZE 1000000 class TxnProto; class DeterministicLockManager { public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); + DeterministicLockManager(deque* ready_txns, Configuration* config); virtual ~DeterministicLockManager() {} virtual int Lock(TxnProto* txn); virtual void Release(const Key& key, TxnProto* txn); @@ -37,7 +37,7 @@ class DeterministicLockManager { hash = hash ^ (key[i]); hash = hash * 16777619; } - return hash % TABLE_SIZE; + return hash % LOCK_TABLE_SIZE; } bool IsLocal(const Key& key) { @@ -60,15 +60,14 @@ class DeterministicLockManager { TxnProto* txn; // Pointer to txn requesting the lock. LockMode mode; // Specifies whether this is a read or write lock request. }; - + struct KeysList { KeysList(Key m, deque* t) : key(m), locksrequest(t) {} Key key; deque* locksrequest; }; - - deque* lock_table_[TABLE_SIZE]; + deque* lock_table_[LOCK_TABLE_SIZE]; // Queue of pointers to transactions that have acquired all locks that // they have requested. 'ready_txns_[key].front()' is the owner of the lock diff --git a/src_calvin/scheduler/deterministic_scheduler.cc b/src_calvin/scheduler/deterministic_scheduler.cc index 71d56536..7d024a2f 100644 --- a/src_calvin/scheduler/deterministic_scheduler.cc +++ b/src_calvin/scheduler/deterministic_scheduler.cc @@ -24,6 +24,7 @@ #include "common/utils.h" #include "common/zmq.hpp" #include "common/connection.h" +#include "common/definitions.hh" #include "backend/storage.h" #include "backend/storage_manager.h" #include "proto/message.pb.h" @@ -35,13 +36,17 @@ // to get COLD_CUTOFF #include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST +#include "common/debug.hh" + +using std::map; using std::pair; using std::string; using std::tr1::unordered_map; using zmq::socket_t; -using std::map; -static void DeleteTxnPtr(void* data, void* hint) { free(data); } +static void DeleteTxnPtr(void* data, void* hint) { + free(data); +} void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); @@ -62,15 +67,17 @@ DeterministicScheduler::DeterministicScheduler(Configuration* conf, Connection* batch_connection, Storage* storage, const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); + : configuration_(conf), + batch_connection_(batch_connection), + storage_(storage), + application_(application) { + ready_txns_ = new std::deque(); lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - + txns_queue = new AtomicQueue(); done_queue = new AtomicQueue(); - for (int i = 0; i < NUM_THREADS; i++) { + for (int i = 0; i < NUM_WORKERS; i++) { message_queues[i] = new AtomicQueue(); } @@ -80,35 +87,31 @@ DeterministicScheduler::DeterministicScheduler(Configuration* conf, cpu_set_t cpuset; pthread_attr_t attr1; pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - + // pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); + CPU_ZERO(&cpuset); - CPU_SET(7, &cpuset); + CPU_SET(LOCK_MANAGER_CORE, &cpuset); pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, reinterpret_cast(this)); - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { + for (int i = 0; i < NUM_WORKERS; i++) { string channel("scheduler"); channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); + thread_connections_[i] = batch_connection_->multiplexer()->NewConnection( + channel, &message_queues[i]); pthread_attr_t attr; pthread_attr_init(&attr); CPU_ZERO(&cpuset); - if (i == 0 || i == 1) - CPU_SET(i, &cpuset); - else - CPU_SET(i+2, &cpuset); + CPU_SET(GET_WORKER_CORE(i), &cpuset); pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); pthread_create(&(threads_[i]), &attr, RunWorkerThread, reinterpret_cast( - new pair(i, this))); + new pair(i, this))); } - } void UnfetchAll(Storage* storage, TxnProto* txn) { @@ -131,6 +134,8 @@ void* DeterministicScheduler::RunWorkerThread(void* arg) { unordered_map active_txns; + PrintCpu("Worker", thread); + // Begin main loop. MessageProto message; while (true) { @@ -146,47 +151,45 @@ void* DeterministicScheduler::RunWorkerThread(void* arg) { scheduler->application_->Execute(txn, manager); delete manager; - scheduler->thread_connections_[thread]-> - UnlinkChannel(IntToString(txn->txn_id())); + scheduler->thread_connections_[thread]->UnlinkChannel( + IntToString(txn->txn_id())); active_txns.erase(message.destination_channel()); // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); + // scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); scheduler->done_queue->Push(txn); } } else { // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); + TxnProto* txn; + bool got_it = scheduler->txns_queue->Pop(&txn); if (got_it == true) { // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } else { - scheduler->thread_connections_[thread]-> - LinkChannel(IntToString(txn->txn_id())); - // There are outstanding remote reads. - active_txns[IntToString(txn->txn_id())] = manager; - } + StorageManager* manager = new StorageManager( + scheduler->configuration_, scheduler->thread_connections_[thread], + scheduler->storage_, txn); + + // Writes occur at this node. + if (manager->ReadyToExecute()) { + // No remote reads. Execute and clean up. + scheduler->application_->Execute(txn, manager); + delete manager; + + // Respond to scheduler; + // scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); + scheduler->done_queue->Push(txn); + } else { + scheduler->thread_connections_[thread]->LinkChannel( + IntToString(txn->txn_id())); + // There are outstanding remote reads. + active_txns[IntToString(txn->txn_id())] = manager; + } } } } return NULL; } -DeterministicScheduler::~DeterministicScheduler() { -} +DeterministicScheduler::~DeterministicScheduler() {} // Returns ptr to heap-allocated unordered_map batches; @@ -213,7 +216,10 @@ MessageProto* GetBatch(int batch_id, Connection* connection) { } void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); + PrintCpu("Lock Manager", 0); + + DeterministicScheduler* scheduler = + reinterpret_cast(arg); // Run main loop. MessageProto message; @@ -224,8 +230,77 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { int pending_txns = 0; int batch_offset = 0; int batch_number = 0; -//int test = 0; + // int test = 0; + + int tasks[Task::Size] = {0}; + + std::string task_names[Task::Size] = {"ProcessDoneTransaction", + "LoadNextBatch", "AdvanceBatch", + "Locking", "ProcessReadyTransaction"}; + + // TxnProto* done_txn; + while (true) { + // if (scheduler->done_queue->Pop(&done_txn)) { + // executing_txns--; + // if (done_txn->writers_size() == 0 || + // rand() % done_txn->writers_size() == 0) + // txns++; + + // // We have received a finished transaction back, release the lock + // scheduler->lock_manager_->Release(done_txn); + // delete done_txn; + // tasks[Task::ProcessDoneTransaction]++; + // goto END; + // } + + // // Have we run out of txns in our batch? Let's get some new ones. + // if (batch_message == NULL) { + // batch_message = GetBatch(batch_number, scheduler->batch_connection_); + // if (batch_message != NULL) + // tasks[Task::LoadNextBatch] += batch_message->data_size(); + // // Done with current batch, get next. + // goto END; + // } else if (batch_offset >= batch_message->data_size()) { + // batch_offset = 0; + // batch_number++; + // delete batch_message; + // batch_message = GetBatch(batch_number, scheduler->batch_connection_); + + // // Current batch has remaining txns, grab up to 10. + // if (batch_message != NULL) + // tasks[Task::AdvanceBatch] += batch_message->data_size(); + // goto END; + // } + + // if (executing_txns + pending_txns < MAX_ACTIVE_TXNS) { + // for (int i = 0; i < LOCK_BATCH_SIZE; i++) { + // if (batch_offset >= batch_message->data_size()) { + // // Oops we ran out of txns in this batch. Stop adding txns for + // now. break; + // } + // TxnProto* txn = new TxnProto(); + // txn->ParseFromString(batch_message->data(batch_offset)); + // batch_offset++; + + // scheduler->lock_manager_->Lock(txn); + // pending_txns++; + // tasks[Task::Locking]++; + // } + // } + + // END: + // // Start executing any and all ready transactions to get them off our + // plate while (!scheduler->ready_txns_->empty()) { + // TxnProto* txn = scheduler->ready_txns_->front(); + // scheduler->ready_txns_->pop_front(); + // pending_txns--; + // executing_txns++; + + // scheduler->txns_queue->Push(txn); + // tasks[Task::ProcessReadyTransaction]++; + // } + TxnProto* done_txn; bool got_it = scheduler->done_queue->Pop(&done_txn); if (got_it == true) { @@ -233,8 +308,9 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { scheduler->lock_manager_->Release(done_txn); executing_txns--; - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) - txns++; + if (done_txn->writers_size() == 0 || + rand() % done_txn->writers_size() == 0) + txns++; delete done_txn; @@ -243,17 +319,16 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { if (batch_message == NULL) { batch_message = GetBatch(batch_number, scheduler->batch_connection_); - // Done with current batch, get next. + // Done with current batch, get next. } else if (batch_offset >= batch_message->data_size()) { batch_offset = 0; batch_number++; delete batch_message; batch_message = GetBatch(batch_number, scheduler->batch_connection_); - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { + // Current batch has remaining txns, grab up to 10. + } else if (executing_txns + pending_txns < MAX_ACTIVE_TXNS) { + for (int i = 0; i < LOCK_BATCH_SIZE; i++) { if (batch_offset >= batch_message->data_size()) { // Oops we ran out of txns in this batch. Stop adding txns for now. break; @@ -265,7 +340,6 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { scheduler->lock_manager_->Lock(txn); pending_txns++; } - } } @@ -282,15 +356,25 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { // Report throughput. if (GetTime() > time + 1) { double total_time = GetTime() - time; + + std::string task_output = "Tasks: "; + for (int i = 0; i < Task::Size; i++) { + task_output.append(task_names[i] + ": " + std::to_string(tasks[i]) + + ", "); + } + std::cout << "Completed " << (static_cast(txns) / total_time) << " txns/sec, " - //<< test<< " for drop speed , " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; + //<< test<< " for drop speed , " + << executing_txns << " executing, " << pending_txns + << " pending, " << "\n" + << task_output << "\n" + << std::flush; // Reset txn count. time = GetTime(); txns = 0; - //test ++; + // test ++; + memset(tasks, 0, sizeof(tasks)); } } return NULL; diff --git a/src_calvin/scheduler/deterministic_scheduler.h b/src_calvin/scheduler/deterministic_scheduler.h index 9ab327f4..4c842511 100644 --- a/src_calvin/scheduler/deterministic_scheduler.h +++ b/src_calvin/scheduler/deterministic_scheduler.h @@ -17,6 +17,7 @@ #include "scheduler/scheduler.h" #include "common/utils.h" +#include "common/definitions.hh" #include "proto/txn.pb.h" #include "proto/message.pb.h" @@ -25,7 +26,7 @@ using std::deque; namespace zmq { class socket_t; class message_t; -} +} // namespace zmq using zmq::socket_t; class Configuration; @@ -34,19 +35,27 @@ class DeterministicLockManager; class Storage; class TxnProto; -#define NUM_THREADS 4 -// #define PREFETCHING - class DeterministicScheduler : public Scheduler { public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); + enum Task { + ProcessDoneTransaction, + LoadNextBatch, + AdvanceBatch, + Locking, + ProcessReadyTransaction, + Size + }; + + DeterministicScheduler(Configuration* conf, + Connection* batch_connection, + Storage* storage, + const Application* application); virtual ~DeterministicScheduler(); private: // Function for starting main loops in a separate pthreads. static void* RunWorkerThread(void* arg); - + static void* LockManagerThread(void* arg); void SendTxnPtr(socket_t* socket, TxnProto* txn); @@ -56,8 +65,8 @@ class DeterministicScheduler : public Scheduler { Configuration* configuration_; // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; + pthread_t threads_[NUM_WORKERS]; + Connection* thread_connections_[NUM_WORKERS]; pthread_t lock_manager_thread_; // Connection for receiving txn batches from sequencer. @@ -65,7 +74,7 @@ class DeterministicScheduler : public Scheduler { // Storage layer used in application execution. Storage* storage_; - + // Application currently being run. const Application* application_; @@ -79,15 +88,14 @@ class DeterministicScheduler : public Scheduler { std::deque* ready_txns_; // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; - + // socket_t* requests_out_; + // socket_t* requests_in_; + // socket_t* responses_out_[NUM_WORKERS]; + // socket_t* responses_in_; + AtomicQueue* txns_queue; AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - + + AtomicQueue* message_queues[NUM_WORKERS]; }; #endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_calvin/scheduler/scheduler.h b/src_calvin/scheduler/scheduler.h index d93465fa..45557a1b 100644 --- a/src_calvin/scheduler/scheduler.h +++ b/src_calvin/scheduler/scheduler.h @@ -14,6 +14,5 @@ class Application; class Scheduler { public: virtual ~Scheduler() {} - }; #endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_calvin/scheduler/serial_scheduler.cc b/src_calvin/scheduler/serial_scheduler.cc index fff24cfd..6b6469c8 100644 --- a/src_calvin/scheduler/serial_scheduler.cc +++ b/src_calvin/scheduler/serial_scheduler.cc @@ -18,11 +18,14 @@ #include "proto/message.pb.h" #include "proto/txn.pb.h" -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} +SerialScheduler::SerialScheduler(Configuration* conf, + Connection* connection, + Storage* storage, + bool checkpointing) + : configuration_(conf), + connection_(connection), + storage_(storage), + checkpointing_(checkpointing) {} SerialScheduler::~SerialScheduler() {} @@ -31,7 +34,7 @@ void SerialScheduler::Run(const Application& application) { TxnProto txn; StorageManager* manager; Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); + connection_->multiplexer()->NewConnection("manager_connection"); int txns = 0; double time = GetTime(); diff --git a/src_calvin/scheduler/serial_scheduler.h b/src_calvin/scheduler/serial_scheduler.h index f99958c6..0fc5e328 100644 --- a/src_calvin/scheduler/serial_scheduler.h +++ b/src_calvin/scheduler/serial_scheduler.h @@ -14,8 +14,10 @@ class Storage; class SerialScheduler : public Scheduler { public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); + SerialScheduler(Configuration* conf, + Connection* connection, + Storage* storage, + bool checkpointing); virtual ~SerialScheduler(); virtual void Run(const Application& application); diff --git a/src_calvin/sequencer/sequencer.cc b/src_calvin/sequencer/sequencer.cc index 56ea7820..12632b81 100644 --- a/src_calvin/sequencer/sequencer.cc +++ b/src_calvin/sequencer/sequencer.cc @@ -18,16 +18,17 @@ #include "common/configuration.h" #include "common/connection.h" #include "common/utils.h" +#include "common/debug.hh" #include "proto/message.pb.h" #include "proto/txn.pb.h" #ifdef PAXOS -# include "paxos/paxos.h" +#include "paxos/paxos.h" #endif using std::map; using std::multimap; -using std::set; using std::queue; +using std::set; #ifdef LATENCY_TEST double sequencer_recv[SAMPLES]; @@ -41,51 +42,49 @@ double worker_end[SAMPLES]; double scheduler_unlock[SAMPLES]; #endif -void* Sequencer::RunSequencerWriter(void *arg) { +void* Sequencer::RunSequencerWriter(void* arg) { reinterpret_cast(arg)->RunWriter(); return NULL; } -void* Sequencer::RunSequencerReader(void *arg) { +void* Sequencer::RunSequencerReader(void* arg) { reinterpret_cast(arg)->RunReader(); return NULL; } -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), storage_(storage), deconstructor_invoked_(false) { +Sequencer::Sequencer(Configuration* conf, + Connection* connection, + Client* client, + Storage* storage) + : epoch_duration_(EPOCH_DURATION), + configuration_(conf), + connection_(connection), + client_(client), + storage_(storage), + deconstructor_invoked_(false) { pthread_mutex_init(&mutex_, NULL); // Start Sequencer main loops running in background thread. -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - + cpu_set_t cpuset; + pthread_attr_t attr_writer; + pthread_attr_init(&attr_writer); + // pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + CPU_ZERO(&cpuset); + CPU_SET(SEQUENCER_WRITER_CORE, &cpuset); + pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); + reinterpret_cast(this)); -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); + CPU_ZERO(&cpuset); + CPU_SET(SEQUENCER_READER_CORE, &cpuset); + pthread_attr_t attr_reader; + pthread_attr_init(&attr_reader); + pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); + reinterpret_cast(this)); } Sequencer::~Sequencer() { @@ -129,6 +128,8 @@ double PrefetchAll(Storage* storage, TxnProto* txn) { #endif void Sequencer::RunWriter() { + PrintCpu("RunWriter", 0); + Spin(1); #ifdef PAXOS @@ -165,33 +166,30 @@ void Sequencer::RunWriter() { string batch_string; batch.set_type(MessageProto::TXN_BATCH); - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; + for (int batch_number = configuration_->this_node_id; !deconstructor_invoked_; batch_number += configuration_->all_nodes.size()) { // Begin epoch. double epoch_start = GetTime(); batch.set_batch_number(batch_number); batch.clear_data(); - - // Collect txn requests for this epoch. int txn_id_offset = 0; while (!deconstructor_invoked_ && GetTime() < epoch_start + epoch_duration_) { // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { + if (batch.data_size() < MAX_LOCK_BATCH_SIZE) { TxnProto* txn; string txn_string; - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); - + client_->GetTxn(&txn, + batch_number * MAX_LOCK_BATCH_SIZE + txn_id_offset); + // Find a bad transaction - if(txn->txn_id() == -1) { + if (txn->txn_id() == -1) { delete txn; continue; } - txn->SerializeToString(&txn_string); batch.add_data(txn_string); txn_id_offset++; @@ -214,6 +212,8 @@ void Sequencer::RunWriter() { } void Sequencer::RunReader() { + PrintCpu("RunReader", 0); + Spin(1); #ifdef PAXOS Paxos paxos(ZOOKEEPER_CONF, true); @@ -318,8 +318,9 @@ void Sequencer::RunReader() { // Report output. if (GetTime() > time + 1) { #ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; + std::cout << "Submitted " << txn_count << " txns in " << batch_count + << " batches,\n" + << std::flush; #endif // Reset txn count. time = GetTime(); diff --git a/src_calvin/sequencer/sequencer.h b/src_calvin/sequencer/sequencer.h index facb677c..e638f332 100644 --- a/src_calvin/sequencer/sequencer.h +++ b/src_calvin/sequencer/sequencer.h @@ -11,20 +11,19 @@ #include #include -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 +#include "common/definitions.hh" -#define MAX_BATCH_SIZE 200 +// #define PAXOS +// #define PREFETCHING #define SAMPLES 100000 #define SAMPLE_RATE 999 -//#define LATENCY_TEST +// #define LATENCY_TEST +using std::queue; using std::set; using std::string; -using std::queue; class Configuration; class Connection; @@ -53,7 +52,9 @@ class Sequencer { public: // The constructor creates background threads and starts the Sequencer's main // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client, + Sequencer(Configuration* conf, + Connection* connection, + Client* client, Storage* storage); // Halts the main loops. @@ -77,8 +78,8 @@ class Sequencer { // Functions to start the Multiplexor's main loops, called in new pthreads by // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); + static void* RunSequencerWriter(void* arg); + static void* RunSequencerReader(void* arg); // Sets '*nodes' to contain the node_id of every node participating in 'txn'. void FindParticipatingNodes(const TxnProto& txn, set* nodes); diff --git a/src_calvin/tests/collapsed_versioned_storage_test.cc b/src_calvin/tests/collapsed_versioned_storage_test.cc index 285b9134..dc3f1bec 100644 --- a/src_calvin/tests/collapsed_versioned_storage_test.cc +++ b/src_calvin/tests/collapsed_versioned_storage_test.cc @@ -76,5 +76,3 @@ int main(int argc, char** argv) { CollapsedVersionedStorageTest(); CheckpointingTest(); } - - diff --git a/src_calvin/tests/configuration_test.cc b/src_calvin/tests/configuration_test.cc index 79a569a5..417605a3 100644 --- a/src_calvin/tests/configuration_test.cc +++ b/src_calvin/tests/configuration_test.cc @@ -30,4 +30,3 @@ int main(int argc, char** argv) { ConfigurationTest_ReadFromFile(); ConfigurationTest_LookupPartition(); } - diff --git a/src_calvin/tests/connection_test.cc b/src_calvin/tests/connection_test.cc index 8bf28f97..aba994ab 100644 --- a/src_calvin/tests/connection_test.cc +++ b/src_calvin/tests/connection_test.cc @@ -158,9 +158,8 @@ TEST(LinkUnlinkChannelTest) { } */ int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); + // InprocTest(); + // RemoteTest(); + // ChannelNotCreatedYetTest(); + // LinkUnlinkChannelTest(); } - diff --git a/src_calvin/tests/deterministic_lock_manager_test.cc b/src_calvin/tests/deterministic_lock_manager_test.cc index be34c6f8..376344d3 100644 --- a/src_calvin/tests/deterministic_lock_manager_test.cc +++ b/src_calvin/tests/deterministic_lock_manager_test.cc @@ -114,9 +114,9 @@ TEST(ThroughputTest) { args.SerializeToString(&args_string); for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); + // txns.push_back(new TxnProto()); + // for (int j = 0; j < 10; j++) + // txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); } @@ -140,8 +140,7 @@ TEST(ThroughputTest) { } int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); + // SimpleLockingTest(); + // LocksReleasedOutOfOrder(); ThroughputTest(); } - diff --git a/src_calvin/tests/deterministic_scheduler_test.cc b/src_calvin/tests/deterministic_scheduler_test.cc index c0139888..2acd2366 100644 --- a/src_calvin/tests/deterministic_scheduler_test.cc +++ b/src_calvin/tests/deterministic_scheduler_test.cc @@ -6,5 +6,4 @@ #include "common/types.h" -int main(int argc, char** argv) { -} +int main(int argc, char** argv) {} diff --git a/src_calvin/tests/fetching_storage_test.cc b/src_calvin/tests/fetching_storage_test.cc index 79a3e4ec..ed63cfed 100644 --- a/src_calvin/tests/fetching_storage_test.cc +++ b/src_calvin/tests/fetching_storage_test.cc @@ -7,7 +7,6 @@ #include "common/testing.h" - TEST(FetchingStorageTest) { system("rm ../db/storage/*"); FetchingStorage* storage = FetchingStorage::BuildStorage(); @@ -32,5 +31,3 @@ TEST(FetchingStorageTest) { int main(int argc, char** argv) { FetchingStorageTest(); } - - diff --git a/src_calvin/tests/microbenchmark_test.cc b/src_calvin/tests/microbenchmark_test.cc index 14b5b8fd..b0059250 100644 --- a/src_calvin/tests/microbenchmark_test.cc +++ b/src_calvin/tests/microbenchmark_test.cc @@ -17,11 +17,12 @@ using std::set; SimpleStorage* actual_storage; Configuration* config; -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) +#define CHECK_OBJECT(KEY, EXPECTED_VALUE) \ + do { \ + Value* actual_value; \ + actual_value = actual_storage->ReadObject(KEY); \ + EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ + } while (0) TEST(MicrobenchmarkTest) { config = new Configuration(0, "common/configuration_test_one_node.conf"); @@ -38,23 +39,21 @@ TEST(MicrobenchmarkTest) { txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); + StorageManager* storage = + new StorageManager(config, connection, actual_storage, txn); microbenchmark.Execute(txn, storage); // Check post-execution storage state. set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) + for (int i = 0; i < Microbenchmark::RW_SET_SIZE; i++) write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { + for (int i = 0; i < microbenchmark.DB_SIZE; i++) { if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); + CHECK_OBJECT(IntToString(i), IntToString(i + 1)); else CHECK_OBJECT(IntToString(i), IntToString(i)); } - - delete storage; delete txn; @@ -69,4 +68,3 @@ TEST(MicrobenchmarkTest) { int main(int argc, char** argv) { MicrobenchmarkTest(); } - diff --git a/src_calvin/tests/paxos_test.cc b/src_calvin/tests/paxos_test.cc old mode 100755 new mode 100644 index 20bdfe09..c57e8dd8 --- a/src_calvin/tests/paxos_test.cc +++ b/src_calvin/tests/paxos_test.cc @@ -18,7 +18,7 @@ int throughput; // Create Paxos object to submit some batches, inserting them into // a globally consistent batch order. -void* Writer(void *arg) { +void* Writer(void* arg) { // Create paxos object. Paxos writer(ZOOKEEPER_CONF, false); @@ -36,7 +36,7 @@ void* Writer(void *arg) { write_number++; // The interal is 0 to 1 msec. srand(50); - usleep((rand()%10)*100); + usleep((rand() % 10) * 100); // Test for 70 seconds. if (end - start > 70) break; @@ -46,7 +46,7 @@ void* Writer(void *arg) { } // Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { +void* Reader(void* arg) { // Create Paxos object. Paxos reader(ZOOKEEPER_CONF, true); diff --git a/src_calvin/tests/profile_test.cc b/src_calvin/tests/profile_test.cc index f55e19f0..57f553fc 100644 --- a/src_calvin/tests/profile_test.cc +++ b/src_calvin/tests/profile_test.cc @@ -8,7 +8,7 @@ int main(int argc, char** argv) { Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); + new Configuration(0, "common/configuration_test_one_node.conf"); CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); TPCC* tpcc = new TPCC(); diff --git a/src_calvin/tests/sequencer_test.cc b/src_calvin/tests/sequencer_test.cc index 12c5b6b9..1fe5eb52 100644 --- a/src_calvin/tests/sequencer_test.cc +++ b/src_calvin/tests/sequencer_test.cc @@ -13,4 +13,3 @@ TEST(SequencerTest) { int main(int argc, char** argv) { SequencerTest(); } - diff --git a/src_calvin/tests/simple_storage_test.cc b/src_calvin/tests/simple_storage_test.cc index 6a855366..c0610cc8 100644 --- a/src_calvin/tests/simple_storage_test.cc +++ b/src_calvin/tests/simple_storage_test.cc @@ -23,5 +23,3 @@ TEST(SimpleStorageTest) { int main(int argc, char** argv) { SimpleStorageTest(); } - - diff --git a/src_calvin/tests/storage_manager_test.cc b/src_calvin/tests/storage_manager_test.cc index be719e5c..1adee300 100644 --- a/src_calvin/tests/storage_manager_test.cc +++ b/src_calvin/tests/storage_manager_test.cc @@ -135,8 +135,7 @@ TEST(TwoNodes) { } int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); + // TODO(alex): Fix these tests! + // SingleNode(); + // TwoNodes(); } - diff --git a/src_calvin/tests/tpcc_test.cc b/src_calvin/tests/tpcc_test.cc index 8dff4592..568b481e 100644 --- a/src_calvin/tests/tpcc_test.cc +++ b/src_calvin/tests/tpcc_test.cc @@ -12,7 +12,7 @@ // We make these global variables to avoid weird pointer passing and code // redundancy Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); + new Configuration(0, "common/configuration_test_one_node.conf"); ConnectionMultiplexer* multiplexer; Connection* connection; SimpleStorage* simple_store; @@ -22,10 +22,9 @@ TPCC* tpcc; TEST(IdGenerationTest) { EXPECT_EQ(tpcc->CheckpointID("w1"), 1); EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); + EXPECT_EQ(tpcc->CheckpointID("c1"), + WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + 1); + EXPECT_EQ(tpcc->CheckpointID("w2si1"), 1000000 + 2 * NUMBER_OF_ITEMS + 1); EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); @@ -217,8 +216,7 @@ TEST(InitializeTest) { for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { char district_key[128]; Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); + snprintf(district_key, sizeof(district_key), "w%dd%d", i, j); district_value = simple_store->ReadObject(district_key); District* dummy_district = new District(); @@ -229,8 +227,7 @@ TEST(InitializeTest) { for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { char customer_key[128]; Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", i, j, k); customer_value = simple_store->ReadObject(customer_key); Customer* dummy_customer = new Customer(); @@ -244,8 +241,7 @@ TEST(InitializeTest) { char item_key[128], stock_key[128]; Value* stock_value; snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); + snprintf(stock_key, sizeof(stock_key), "%ss%s", warehouse_key, item_key); stock_value = simple_store->ReadObject(stock_key); Stock* dummy_stock = new Stock(); @@ -256,14 +252,14 @@ TEST(InitializeTest) { // Expect all items to be there for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; + char item_key[128]; + Value item_value; + snprintf(item_key, sizeof(item_key), "i%d", i); + item_value = *(tpcc->GetItem(string(item_key))); + + Item* dummy_item = new Item(); + EXPECT_TRUE(dummy_item->ParseFromString(item_value)); + delete dummy_item; } END; @@ -292,11 +288,11 @@ TEST(NewOrderTest) { txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); + StorageManager* storage = + new StorageManager(config, connection, simple_store, txn); // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); + District* district = new District(); Value* district_value; district_value = storage->ReadObject(txn->read_write_set(0)); assert(district->ParseFromString(*district_value)); @@ -403,15 +399,15 @@ TEST(NewOrderTest) { // Check the order line // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); + // Value* order_line_value; + // OrderLine* order_line = new OrderLine(); + // order_line_value = storage->ReadObject(txn->write_set(i)); + // EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); + // EXPECT_EQ(order_line->amount(), item->price() * + // txn_args->quantities(i)); EXPECT_EQ(order_line->number(), i); // Free memory -// delete order_line; + // delete order_line; delete item; delete stock; } @@ -447,25 +443,25 @@ TEST(PaymentTest) { txn->add_read_set(txn->write_set(0)); txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); + StorageManager* storage = + new StorageManager(config, connection, simple_store, txn); // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); + Warehouse* warehouse = new Warehouse(); Value* warehouse_value; warehouse_value = storage->ReadObject(txn->read_write_set(0)); assert(warehouse->ParseFromString(*warehouse_value)); int old_warehouse_year_to_date = warehouse->year_to_date(); // Prefetch district - District *district = new District(); + District* district = new District(); Value* district_value; district_value = storage->ReadObject(txn->read_write_set(1)); assert(district->ParseFromString(*district_value)); int old_district_year_to_date = district->year_to_date(); // Preetch customer - Customer *customer = new Customer(); + Customer* customer = new Customer(); Value* customer_value; customer_value = storage->ReadObject(txn->read_write_set(2)); assert(customer->ParseFromString(*customer_value)); @@ -488,10 +484,10 @@ TEST(PaymentTest) { assert(customer->ParseFromString(*customer_value)); // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); + EXPECT_EQ(warehouse->year_to_date(), + old_warehouse_year_to_date + txn_args->amount()); + EXPECT_EQ(district->year_to_date(), + old_district_year_to_date + txn_args->amount()); EXPECT_EQ(customer->year_to_date_payment(), old_customer_year_to_date_payment + txn_args->amount()); EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); @@ -574,4 +570,3 @@ int main(int argc, char** argv) { return 0; } - diff --git a/src_calvin/tests/types_test.cc b/src_calvin/tests/types_test.cc index ef5bc102..485b9d24 100644 --- a/src_calvin/tests/types_test.cc +++ b/src_calvin/tests/types_test.cc @@ -5,7 +5,7 @@ #include "common/testing.h" TEST(PackSignedIntTest) { - int8 i1 = 65; + int8 i1 = 65; int16 i2 = -2551; int32 i3 = 0; int64 i4 = -2551255125512551; @@ -19,7 +19,7 @@ TEST(PackSignedIntTest) { } TEST(PackUnsignedIntTest) { - uint8 u1 = 251; + uint8 u1 = 251; uint16 u2 = 2551; uint32 u3 = 0; uint64 u4 = 2551255125512551; diff --git a/src_calvin_3_partitions/Makefile b/src_calvin_3_partitions/Makefile deleted file mode 100755 index e3e6e17d..00000000 --- a/src_calvin_3_partitions/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_calvin_3_partitions/applications/Makefile.inc b/src_calvin_3_partitions/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_calvin_3_partitions/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_3_partitions/applications/application.h b/src_calvin_3_partitions/applications/application.h deleted file mode 100755 index 047e8513..00000000 --- a/src_calvin_3_partitions/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, - Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_calvin_3_partitions/applications/microbenchmark.cc b/src_calvin_3_partitions/applications/microbenchmark.cc deleted file mode 100755 index 7d87240d..00000000 --- a/src_calvin_3_partitions/applications/microbenchmark.cc +++ /dev/null @@ -1,176 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// Create a non-dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2, int part3) { - assert(part1 != part2 || nparts == 1); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - int hotkey3 = part3 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - txn->add_read_write_set(IntToString(hotkey3)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - 3, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part3); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - *val = IntToString(StringToInt(*val) + 1); - // Not necessary since storage already has a pointer to val. - // storage->PutObject(txn->read_write_set(i), val); - - // The following code is for microbenchmark "long" transaction, uncomment it if for "long" transaction - /**int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - }**/ - - } - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, - Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { -#ifdef PREFETCHING - if (i % 10000 == 0) - std::cout << i << std::endl; - storage->Prefetch(IntToString(i), &wait_time); - storage->PutObject(IntToString(i), new Value(IntToString(i))); - if (i > COLD_CUTOFF) { - storage->Unfetch(IntToString(i)); - if (i % 10 == 0) - std::cout << i << std::endl; - } -#else - storage->PutObject(IntToString(i), new Value(IntToString(i))); -#endif - } - } -} - diff --git a/src_calvin_3_partitions/applications/microbenchmark.h b/src_calvin_3_partitions/applications/microbenchmark.h deleted file mode 100755 index 74d24bf3..00000000 --- a/src_calvin_3_partitions/applications/microbenchmark.h +++ /dev/null @@ -1,55 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2, int part3); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_calvin_3_partitions/applications/tpcc.cc b/src_calvin_3_partitions/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_calvin_3_partitions/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_calvin_3_partitions/applications/tpcc.h b/src_calvin_3_partitions/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_calvin_3_partitions/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_calvin_3_partitions/backend/collapsed_versioned_storage.cc b/src_calvin_3_partitions/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_calvin_3_partitions/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_calvin_3_partitions/backend/fetching_storage.cc b/src_calvin_3_partitions/backend/fetching_storage.cc deleted file mode 100755 index f5514372..00000000 --- a/src_calvin_3_partitions/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_calvin_3_partitions/backend/fetching_storage.h b/src_calvin_3_partitions/backend/fetching_storage.h deleted file mode 100755 index 273dc04c..00000000 --- a/src_calvin_3_partitions/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_calvin_3_partitions/backend/simple_storage.cc b/src_calvin_3_partitions/backend/simple_storage.cc deleted file mode 100755 index 46237c8f..00000000 --- a/src_calvin_3_partitions/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_calvin_3_partitions/backend/simple_storage.h b/src_calvin_3_partitions/backend/simple_storage.h deleted file mode 100755 index 9fb2ee24..00000000 --- a/src_calvin_3_partitions/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_calvin_3_partitions/backend/storage.h b/src_calvin_3_partitions/backend/storage.h deleted file mode 100755 index a793360a..00000000 --- a/src_calvin_3_partitions/backend/storage.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_calvin_3_partitions/backend/storage_manager.cc b/src_calvin_3_partitions/backend/storage_manager.cc deleted file mode 100755 index 7de4ff40..00000000 --- a/src_calvin_3_partitions/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == configuration_->this_node_id) - reader = true; - } - - if (reader) { - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != configuration_->this_node_id) { - message.set_destination_node(txn->writers(i)); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == configuration_->this_node_id) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_calvin_3_partitions/backend/storage_manager.h b/src_calvin_3_partitions/backend/storage_manager.h deleted file mode 100755 index 6d02dfc4..00000000 --- a/src_calvin_3_partitions/backend/storage_manager.h +++ /dev/null @@ -1,87 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_calvin_3_partitions/backend/versioned_storage.h b/src_calvin_3_partitions/backend/versioned_storage.h deleted file mode 100755 index 24c6223b..00000000 --- a/src_calvin_3_partitions/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_calvin_3_partitions/common/configuration.cc b/src_calvin_3_partitions/common/configuration.cc deleted file mode 100755 index a55c0b81..00000000 --- a/src_calvin_3_partitions/common/configuration.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_calvin_3_partitions/common/configuration.h b/src_calvin_3_partitions/common/configuration.h deleted file mode 100755 index d7ce7fb5..00000000 --- a/src_calvin_3_partitions/common/configuration.h +++ /dev/null @@ -1,95 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/types.h" - - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - -struct Node { - // Globally unique node identifier. - int node_id; - int replica_id; - int partition_id; - - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_calvin_3_partitions/common/connection.cc b/src_calvin_3_partitions/common/connection.cc deleted file mode 100755 index d6a1e7d5..00000000 --- a/src_calvin_3_partitions/common/connection.cc +++ /dev/null @@ -1,379 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - - if (message.type() == MessageProto::READ_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_calvin_3_partitions/common/connection.h b/src_calvin_3_partitions/common/connection.h deleted file mode 100755 index 99fa7c0e..00000000 --- a/src_calvin_3_partitions/common/connection.h +++ /dev/null @@ -1,197 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_calvin_3_partitions/common/testing.h b/src_calvin_3_partitions/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_calvin_3_partitions/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_calvin_3_partitions/common/types.h b/src_calvin_3_partitions/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_calvin_3_partitions/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_calvin_3_partitions/common/zmq.hpp b/src_calvin_3_partitions/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_calvin_3_partitions/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_calvin_3_partitions/deployment/cluster.cc b/src_calvin_3_partitions/deployment/cluster.cc deleted file mode 100755 index c04f82b9..00000000 --- a/src_calvin_3_partitions/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_calvin_3_partitions/deployment/main.cc b/src_calvin_3_partitions/deployment/main.cc deleted file mode 100755 index e4b3ead9..00000000 --- a/src_calvin_3_partitions/deployment/main.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other1; - int other2; - do { - other1 = rand() % config_->all_nodes.size(); - } while (other1 == config_->this_node_id); - - do { - other2 = rand() % config_->all_nodes.size(); - } while (other2 == config_->this_node_id || other2 == other1); - - - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other1, other2); - } else { - // Single-partition txn. - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, - storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_calvin_3_partitions/obj/.deps b/src_calvin_3_partitions/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_calvin_3_partitions/obj/common/configuration.d b/src_calvin_3_partitions/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_calvin_3_partitions/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_calvin_3_partitions/paxos/paxos.cc b/src_calvin_3_partitions/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_calvin_3_partitions/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_calvin_3_partitions/paxos/paxos.h b/src_calvin_3_partitions/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_calvin_3_partitions/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_calvin_3_partitions/proto/message.proto b/src_calvin_3_partitions/proto/message.proto deleted file mode 100755 index 8b43a929..00000000 --- a/src_calvin_3_partitions/proto/message.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; -} - diff --git a/src_calvin_3_partitions/scheduler/deterministic_lock_manager.cc b/src_calvin_3_partitions/scheduler/deterministic_lock_manager.cc deleted file mode 100644 index 15d002e3..00000000 --- a/src_calvin_3_partitions/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - deque* requests = it1->locksrequest; - - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - - } -} - diff --git a/src_calvin_3_partitions/scheduler/deterministic_scheduler.cc b/src_calvin_3_partitions/scheduler/deterministic_scheduler.cc deleted file mode 100755 index e476916f..00000000 --- a/src_calvin_3_partitions/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,300 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/zmq.hpp" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" -#include "applications/tpcc.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using zmq::socket_t; -using std::map; - -static void DeleteTxnPtr(void* data, void* hint) { free(data); } - -void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { - TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); - *txn_ptr = txn; - zmq::message_t msg(txn_ptr, sizeof(*txn_ptr), DeleteTxnPtr, NULL); - socket->send(msg); -} - -TxnProto* DeterministicScheduler::GetTxnPtr(socket_t* socket, - zmq::message_t* msg) { - if (!socket->recv(msg, ZMQ_NOBLOCK)) - return NULL; - TxnProto* txn = *reinterpret_cast(msg->data()); - return txn; -} - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - Connection* batch_connection, - Storage* storage, - const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - txns_queue = new AtomicQueue(); - done_queue = new AtomicQueue(); - - for (int i = 0; i < NUM_THREADS; i++) { - message_queues[i] = new AtomicQueue(); - } - -Spin(2); - - // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(7, &cpuset); - pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); - pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, - reinterpret_cast(this)); - - -// pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { - string channel("scheduler"); - channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); - -pthread_attr_t attr; -pthread_attr_init(&attr); -CPU_ZERO(&cpuset); -if (i == 0 || i == 1) -CPU_SET(i, &cpuset); -else -CPU_SET(i+2, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, - reinterpret_cast( - new pair(i, this))); - } - -} - -void UnfetchAll(Storage* storage, TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (StringToInt(txn->read_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_set(i)); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (StringToInt(txn->read_write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_write_set(i)); - for (int i = 0; i < txn->write_set_size(); i++) - if (StringToInt(txn->write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->write_set(i)); -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - int thread = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - unordered_map active_txns; - - // Begin main loop. - MessageProto message; - while (true) { - bool got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - // Remote read result. - assert(message.type() == MessageProto::READ_RESULT); - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - // Execute and clean up. - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - delete manager; - - scheduler->thread_connections_[thread]-> - UnlinkChannel(IntToString(txn->txn_id())); - active_txns.erase(message.destination_channel()); - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } - } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { - // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } else { - scheduler->thread_connections_[thread]-> - LinkChannel(IntToString(txn->txn_id())); - // There are outstanding remote reads. - active_txns[IntToString(txn->txn_id())] = manager; - } - } - } - } - return NULL; -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -MessageProto* GetBatch(int batch_id, Connection* connection) { - if (batches.count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[batch_id]; - batches.erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - - // Run main loop. - MessageProto message; - MessageProto* batch_message = NULL; - int txns = 0; - double time = GetTime(); - int executing_txns = 0; - int pending_txns = 0; - int batch_offset = 0; - int batch_number = 0; -//int test = 0; - while (true) { - TxnProto* done_txn; - bool got_it = scheduler->done_queue->Pop(&done_txn); - if (got_it == true) { - // We have received a finished transaction back, release the lock - scheduler->lock_manager_->Release(done_txn); - executing_txns--; - - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) - txns++; - delete done_txn; - - } else { - // Have we run out of txns in our batch? Let's get some new ones. - if (batch_message == NULL) { - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Done with current batch, get next. - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { - if (batch_offset >= batch_message->data_size()) { - // Oops we ran out of txns in this batch. Stop adding txns for now. - break; - } - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - - scheduler->lock_manager_->Lock(txn); - pending_txns++; - } - - } - } - - // Start executing any and all ready transactions to get them off our plate - while (!scheduler->ready_txns_->empty()) { - TxnProto* txn = scheduler->ready_txns_->front(); - scheduler->ready_txns_->pop_front(); - pending_txns--; - executing_txns++; - - scheduler->txns_queue->Push(txn); - //scheduler->SendTxnPtr(scheduler->requests_out_, txn); - - } - - // Report throughput. - if (GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout << "Completed " << (static_cast(txns) / total_time) - << " txns/sec, " - //<< test<< " for drop speed , " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - //test ++; - } - } - return NULL; -} diff --git a/src_calvin_3_partitions/scheduler/deterministic_scheduler.h b/src_calvin_3_partitions/scheduler/deterministic_scheduler.h deleted file mode 100755 index 33fcf55d..00000000 --- a/src_calvin_3_partitions/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include - -#include "scheduler/scheduler.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; - -namespace zmq { -class socket_t; -class message_t; -} -using zmq::socket_t; - -class Configuration; -class Connection; -class DeterministicLockManager; -class Storage; -class TxnProto; - -#define NUM_THREADS 4 -// #define PREFETCHING - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - static void* LockManagerThread(void* arg); - - void SendTxnPtr(socket_t* socket, TxnProto* txn); - TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; - - pthread_t lock_manager_thread_; - // Connection for receiving txn batches from sequencer. - Connection* batch_connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Application currently being run. - const Application* application_; - - // The per-node lock manager tracks what transactions have temporary ownership - // of what database objects, allowing the scheduler to track LOCAL conflicts - // and enforce equivalence to transaction orders. - DeterministicLockManager* lock_manager_; - - // Queue of transaction ids of transactions that have acquired all locks that - // they have requested. - std::deque* ready_txns_; - - // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; - - AtomicQueue* txns_queue; - AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_calvin_3_partitions/scheduler/scheduler.h b/src_calvin_3_partitions/scheduler/scheduler.h deleted file mode 100755 index d93465fa..00000000 --- a/src_calvin_3_partitions/scheduler/scheduler.h +++ /dev/null @@ -1,19 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} - -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_calvin_3_partitions/scheduler/serial_scheduler.cc b/src_calvin_3_partitions/scheduler/serial_scheduler.cc deleted file mode 100755 index 1d2c9bfe..00000000 --- a/src_calvin_3_partitions/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_calvin_3_partitions/scheduler/serial_scheduler.h b/src_calvin_3_partitions/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_calvin_3_partitions/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_calvin_3_partitions/sequencer/sequencer.cc b/src_calvin_3_partitions/sequencer/sequencer.cc deleted file mode 100755 index 222f4f85..00000000 --- a/src_calvin_3_partitions/sequencer/sequencer.cc +++ /dev/null @@ -1,364 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), storage_(storage), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - // Start Sequencer main loops running in background thread. - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -#ifdef PREFETCHING -double PrefetchAll(Storage* storage, TxnProto* txn) { - double max_wait_time = 0; - double wait_time = 0; - for (int i = 0; i < txn->read_set_size(); i++) { - storage->Prefetch(txn->read_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - storage->Prefetch(txn->read_write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->write_set_size(); i++) { - storage->Prefetch(txn->write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) - prefetch_cold[txn->txn_id() / SAMPLE_RATE] = max_wait_time; -#endif - return max_wait_time; -} -#endif - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - -#ifdef PREFETCHING - multimap fetching_txns; -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - std::cout << "Starting sequencer.\n" << std::flush; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); - - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - -#ifdef PREFETCHING - // Include txn requests from earlier that have now had time to prefetch. - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - multimap::iterator it = fetching_txns.begin(); - if (it == fetching_txns.end() || it->first > GetTime() || - batch.data_size() >= MAX_BATCH_SIZE) { - break; - } - TxnProto* txn = it->second; - fetching_txns.erase(it); - string txn_string; - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } -#endif - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) { - sequencer_recv[txn->txn_id() / SAMPLE_RATE] = - epoch_start - + epoch_duration_ * (static_cast(rand()) / RAND_MAX); - } -#endif -#ifdef PREFETCHING - double wait_time = PrefetchAll(storage_, txn); - if (wait_time > 0) { - fetching_txns.insert(std::make_pair(epoch_start + wait_time, txn)); - } else { - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } -#else - if(txn->txn_id() == -1) { - delete txn; - continue; - } - - - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; -#endif - } - } - - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - - // Set up batch messages for each system node. - map batches; - for (map::iterator it = configuration_->all_nodes.begin(); - it != configuration_->all_nodes.end(); ++it) { - batches[it->first].set_destination_channel("scheduler_"); - batches[it->first].set_destination_node(it->first); - batches[it->first].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif - - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - } while (!got_batch); -#endif - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - -#ifdef LATENCY_TEST - if (txn.txn_id() % SAMPLE_RATE == 0) - watched_txn = txn.txn_id(); -#endif - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - - bytes txn_data; - txn.SerializeToString(&txn_data); - - // Compute union of 'readers' and 'writers' (store in 'readers'). - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - // Insert txn into appropriate batches. - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - batches[*it].add_data(txn_data); - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - -#ifdef LATENCY_TEST - if (watched_txn != -1) { - sequencer_send[watched_txn] = GetTime(); - watched_txn = -1; - } -#endif - - // Report output. - if (GetTime() > time + 1) { -#ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; -#endif - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_calvin_3_partitions/sequencer/sequencer.h b/src_calvin_3_partitions/sequencer/sequencer.h deleted file mode 100755 index bf7d7e64..00000000 --- a/src_calvin_3_partitions/sequencer/sequencer.h +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 56 - -#define SAMPLES 100000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST - -using std::set; -using std::string; -using std::queue; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client, - Storage* storage); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Pointer to this node's storage object, for prefetching. - Storage* storage_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_calvin_3_partitions/tests/collapsed_versioned_storage_test.cc b/src_calvin_3_partitions/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_calvin_3_partitions/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_calvin_3_partitions/tests/configuration_test.cc b/src_calvin_3_partitions/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_calvin_3_partitions/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_calvin_3_partitions/tests/deterministic_lock_manager_test.cc b/src_calvin_3_partitions/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_calvin_3_partitions/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_calvin_3_partitions/tests/deterministic_scheduler_test.cc b/src_calvin_3_partitions/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_calvin_3_partitions/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_calvin_3_partitions/tests/microbenchmark_test.cc b/src_calvin_3_partitions/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_calvin_3_partitions/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_calvin_3_partitions/tests/profile_test.cc b/src_calvin_3_partitions/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_calvin_3_partitions/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_calvin_3_partitions/tests/sequencer_test.cc b/src_calvin_3_partitions/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_calvin_3_partitions/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_calvin_3_partitions/tests/simple_storage_test.cc b/src_calvin_3_partitions/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_calvin_3_partitions/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_calvin_3_partitions/tests/storage_manager_test.cc b/src_calvin_3_partitions/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_calvin_3_partitions/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_calvin_3_partitions/tests/tpcc_test.cc b/src_calvin_3_partitions/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_calvin_3_partitions/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_calvin_3_partitions/tests/types_test.cc b/src_calvin_3_partitions/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_calvin_3_partitions/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_calvin_4_partitions/Makefile.template b/src_calvin_4_partitions/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_calvin_4_partitions/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_calvin_4_partitions/applications/Makefile.inc b/src_calvin_4_partitions/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_calvin_4_partitions/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/applications/application.h b/src_calvin_4_partitions/applications/application.h deleted file mode 100755 index 047e8513..00000000 --- a/src_calvin_4_partitions/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, - Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_calvin_4_partitions/applications/microbenchmark.cc b/src_calvin_4_partitions/applications/microbenchmark.cc deleted file mode 100755 index c058d068..00000000 --- a/src_calvin_4_partitions/applications/microbenchmark.cc +++ /dev/null @@ -1,186 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// Create a non-dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2, int part3, int part4) { - assert(part1 != part2 || nparts == 1); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - - // Add four hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - int hotkey3 = part3 + nparts * (rand() % hot_records); - int hotkey4 = part4 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - txn->add_read_write_set(IntToString(hotkey3)); - txn->add_read_write_set(IntToString(hotkey4)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 1, - nparts * hot_records, - nparts * kDBSize, - part3); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 1, - nparts * hot_records, - nparts * kDBSize, - part4); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - *val = IntToString(StringToInt(*val) + 1); - // Not necessary since storage already has a pointer to val. - // storage->PutObject(txn->read_write_set(i), val); - - // The following code is for microbenchmark "long" transaction, uncomment it if for "long" transaction - /**int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - }**/ - - } - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, - Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { -#ifdef PREFETCHING - if (i % 10000 == 0) - std::cout << i << std::endl; - storage->Prefetch(IntToString(i), &wait_time); - storage->PutObject(IntToString(i), new Value(IntToString(i))); - if (i > COLD_CUTOFF) { - storage->Unfetch(IntToString(i)); - if (i % 10 == 0) - std::cout << i << std::endl; - } -#else - storage->PutObject(IntToString(i), new Value(IntToString(i))); -#endif - } - } -} - diff --git a/src_calvin_4_partitions/applications/microbenchmark.h b/src_calvin_4_partitions/applications/microbenchmark.h deleted file mode 100755 index c0b4f3ae..00000000 --- a/src_calvin_4_partitions/applications/microbenchmark.h +++ /dev/null @@ -1,55 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2, int part3, int part4); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_calvin_4_partitions/applications/tpcc.cc b/src_calvin_4_partitions/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_calvin_4_partitions/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_calvin_4_partitions/applications/tpcc.h b/src_calvin_4_partitions/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_calvin_4_partitions/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_calvin_4_partitions/backend/Makefile.inc b/src_calvin_4_partitions/backend/Makefile.inc deleted file mode 100755 index edcf67bd..00000000 --- a/src_calvin_4_partitions/backend/Makefile.inc +++ /dev/null @@ -1,28 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/checkpointable_storage.cc \ - backend/collapsed_versioned_storage.cc \ - backend/fetching_storage.cc \ - backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/backend/collapsed_versioned_storage.cc b/src_calvin_4_partitions/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_calvin_4_partitions/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_calvin_4_partitions/backend/collapsed_versioned_storage.h b/src_calvin_4_partitions/backend/collapsed_versioned_storage.h deleted file mode 100755 index 249a40d6..00000000 --- a/src_calvin_4_partitions/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_calvin_4_partitions/backend/simple_storage.cc b/src_calvin_4_partitions/backend/simple_storage.cc deleted file mode 100755 index 46237c8f..00000000 --- a/src_calvin_4_partitions/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_calvin_4_partitions/backend/simple_storage.h b/src_calvin_4_partitions/backend/simple_storage.h deleted file mode 100755 index 9fb2ee24..00000000 --- a/src_calvin_4_partitions/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_calvin_4_partitions/backend/storage.h b/src_calvin_4_partitions/backend/storage.h deleted file mode 100755 index a793360a..00000000 --- a/src_calvin_4_partitions/backend/storage.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_calvin_4_partitions/backend/storage_manager.cc b/src_calvin_4_partitions/backend/storage_manager.cc deleted file mode 100755 index 7de4ff40..00000000 --- a/src_calvin_4_partitions/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == configuration_->this_node_id) - reader = true; - } - - if (reader) { - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != configuration_->this_node_id) { - message.set_destination_node(txn->writers(i)); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == configuration_->this_node_id) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_calvin_4_partitions/backend/storage_manager.h b/src_calvin_4_partitions/backend/storage_manager.h deleted file mode 100755 index 6d02dfc4..00000000 --- a/src_calvin_4_partitions/backend/storage_manager.h +++ /dev/null @@ -1,87 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_calvin_4_partitions/common/Makefile.inc b/src_calvin_4_partitions/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_calvin_4_partitions/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/common/configuration.cc b/src_calvin_4_partitions/common/configuration.cc deleted file mode 100755 index a55c0b81..00000000 --- a/src_calvin_4_partitions/common/configuration.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_calvin_4_partitions/common/configuration_test.conf b/src_calvin_4_partitions/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_calvin_4_partitions/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_calvin_4_partitions/common/configuration_test_one_node.conf b/src_calvin_4_partitions/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_calvin_4_partitions/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_calvin_4_partitions/common/connection.cc b/src_calvin_4_partitions/common/connection.cc deleted file mode 100755 index d6a1e7d5..00000000 --- a/src_calvin_4_partitions/common/connection.cc +++ /dev/null @@ -1,379 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - - if (message.type() == MessageProto::READ_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_calvin_4_partitions/common/test_template.cc b/src_calvin_4_partitions/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_calvin_4_partitions/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_calvin_4_partitions/common/testing.h b/src_calvin_4_partitions/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_calvin_4_partitions/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_calvin_4_partitions/common/types.h b/src_calvin_4_partitions/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_calvin_4_partitions/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_calvin_4_partitions/common/utils.h b/src_calvin_4_partitions/common/utils.h deleted file mode 100755 index ea1021cb..00000000 --- a/src_calvin_4_partitions/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_calvin_4_partitions/common/zmq.hpp b/src_calvin_4_partitions/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_calvin_4_partitions/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_calvin_4_partitions/deployment/Makefile.inc b/src_calvin_4_partitions/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_calvin_4_partitions/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_calvin_4_partitions/deployment/addhosts.pl b/src_calvin_4_partitions/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_calvin_4_partitions/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_calvin_4_partitions/deployment/cluster.cc b/src_calvin_4_partitions/deployment/cluster.cc deleted file mode 100755 index c04f82b9..00000000 --- a/src_calvin_4_partitions/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_calvin_4_partitions/deployment/main.cc b/src_calvin_4_partitions/deployment/main.cc deleted file mode 100755 index c9fe4c14..00000000 --- a/src_calvin_4_partitions/deployment/main.cc +++ /dev/null @@ -1,191 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other1; - int other2; - int other3; - do { - other1 = rand() % config_->all_nodes.size(); - } while (other1 == config_->this_node_id); - - do { - other2 = rand() % config_->all_nodes.size(); - } while (other2 == config_->this_node_id || other2 == other1); - - do { - other3 = rand() % config_->all_nodes.size(); - } while (other3 == config_->this_node_id || other3 == other1 || other3 == other2); - - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other1, other2, other3); - } else { - // Single-partition txn. - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, - storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_calvin_4_partitions/deployment/makeconf.pl b/src_calvin_4_partitions/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_calvin_4_partitions/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_calvin_4_partitions/deployment/portfile b/src_calvin_4_partitions/deployment/portfile deleted file mode 100755 index 7436f00b..00000000 --- a/src_calvin_4_partitions/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -54564 diff --git a/src_calvin_4_partitions/deployment/run.pbs b/src_calvin_4_partitions/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_calvin_4_partitions/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_calvin_4_partitions/deployment/test.conf b/src_calvin_4_partitions/deployment/test.conf deleted file mode 100755 index 36bc2882..00000000 --- a/src_calvin_4_partitions/deployment/test.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:1:12:128.36.232.12:51261 #rhino -node2=0:2:12:128.36.232.15:51262 # gator -node3=0:3:12:128.36.232.20:51261 #macaw -node4=0:4:12:128.36.232.19:51262 # peacock -node5=0:5:12:128.36.232.34:51261 # perch -node6=0:6:12:128.36.232.14:51262 # jaguar -node7=0:7:12:128.36.232.16:51261 # hippo -node8=0:8:12:128.36.232.39:51261 #viper -node9=0:9:12:128.36.232.7:51261 #hornet -node10=0:10:12:128.36.232.8:51261 #cricket -node11=0:11:12:128.36.232.28:51261 #termite -node12=0:12:12:128.36.232.5:51261 #cicada -node13=0:13:12:128.36.232.6:51261 #bumblebee -node14=0:14:12:128.36.232.44:51261 #frog -node15=0:15:12:128.36.232.50:51261 #tick -node16=0:16:12:128.36.232.42:51261 #turtle -node17=0:17:12:128.36.232.35:51261 #aphid -node18=0:18:12:128.36.232.41:51261 #chameleon -node19=0:19:12:128.36.232.38:51261 #rattlesnake -node20=0:20:12:128.36.232.45:51261 #scorpion -node21=0:21:12:128.36.232.29:51261 #cardinal -node22=0:22:12:128.36.232.24:51261 #swan -node23=0:23:12:128.36.232.27:51261 #monkey - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_calvin_4_partitions/obj/.deps b/src_calvin_4_partitions/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_calvin_4_partitions/obj/common/configuration.d b/src_calvin_4_partitions/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_calvin_4_partitions/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_calvin_4_partitions/paxos/Makefile.inc b/src_calvin_4_partitions/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_calvin_4_partitions/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/paxos/paxos.cc b/src_calvin_4_partitions/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_calvin_4_partitions/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_calvin_4_partitions/paxos/paxos.h b/src_calvin_4_partitions/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_calvin_4_partitions/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_calvin_4_partitions/paxos/zookeeper.conf b/src_calvin_4_partitions/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_calvin_4_partitions/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_calvin_4_partitions/proto/Makefile.inc b/src_calvin_4_partitions/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_calvin_4_partitions/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_calvin_4_partitions/proto/message.proto b/src_calvin_4_partitions/proto/message.proto deleted file mode 100755 index 8b43a929..00000000 --- a/src_calvin_4_partitions/proto/message.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; -} - diff --git a/src_calvin_4_partitions/proto/tpcc.proto b/src_calvin_4_partitions/proto/tpcc.proto deleted file mode 100755 index 50088661..00000000 --- a/src_calvin_4_partitions/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_calvin_4_partitions/proto/tpcc_args.proto b/src_calvin_4_partitions/proto/tpcc_args.proto deleted file mode 100755 index 248051c8..00000000 --- a/src_calvin_4_partitions/proto/tpcc_args.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - - optional int32 order_number = 41; - - optional int32 lastest_order_number = 42; - optional int32 threshold = 51; -} diff --git a/src_calvin_4_partitions/proto/txn.proto b/src_calvin_4_partitions/proto/txn.proto deleted file mode 100755 index 464a8719..00000000 --- a/src_calvin_4_partitions/proto/txn.proto +++ /dev/null @@ -1,57 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; -} - diff --git a/src_calvin_4_partitions/scheduler/Makefile.inc b/src_calvin_4_partitions/scheduler/Makefile.inc deleted file mode 100755 index f5678dcc..00000000 --- a/src_calvin_4_partitions/scheduler/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc \ - scheduler/serial_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/scheduler/deterministic_lock_manager.cc b/src_calvin_4_partitions/scheduler/deterministic_lock_manager.cc deleted file mode 100644 index 15d002e3..00000000 --- a/src_calvin_4_partitions/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - deque* requests = it1->locksrequest; - - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - - } -} - diff --git a/src_calvin_4_partitions/scheduler/deterministic_lock_manager.h b/src_calvin_4_partitions/scheduler/deterministic_lock_manager.h deleted file mode 100755 index aadd3688..00000000 --- a/src_calvin_4_partitions/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_calvin_4_partitions/scheduler/deterministic_scheduler.h b/src_calvin_4_partitions/scheduler/deterministic_scheduler.h deleted file mode 100755 index 9ab327f4..00000000 --- a/src_calvin_4_partitions/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include - -#include "scheduler/scheduler.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; - -namespace zmq { -class socket_t; -class message_t; -} -using zmq::socket_t; - -class Configuration; -class Connection; -class DeterministicLockManager; -class Storage; -class TxnProto; - -#define NUM_THREADS 4 -// #define PREFETCHING - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - static void* LockManagerThread(void* arg); - - void SendTxnPtr(socket_t* socket, TxnProto* txn); - TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; - - pthread_t lock_manager_thread_; - // Connection for receiving txn batches from sequencer. - Connection* batch_connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Application currently being run. - const Application* application_; - - // The per-node lock manager tracks what transactions have temporary ownership - // of what database objects, allowing the scheduler to track LOCAL conflicts - // and enforce equivalence to transaction orders. - DeterministicLockManager* lock_manager_; - - // Queue of transaction ids of transactions that have acquired all locks that - // they have requested. - std::deque* ready_txns_; - - // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; - - AtomicQueue* txns_queue; - AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_calvin_4_partitions/scheduler/lock_manager.h b/src_calvin_4_partitions/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_calvin_4_partitions/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_calvin_4_partitions/scheduler/scheduler.h b/src_calvin_4_partitions/scheduler/scheduler.h deleted file mode 100755 index d93465fa..00000000 --- a/src_calvin_4_partitions/scheduler/scheduler.h +++ /dev/null @@ -1,19 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} - -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_calvin_4_partitions/scheduler/serial_scheduler.cc b/src_calvin_4_partitions/scheduler/serial_scheduler.cc deleted file mode 100755 index 3e532069..00000000 --- a/src_calvin_4_partitions/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_calvin_4_partitions/scheduler/serial_scheduler.h b/src_calvin_4_partitions/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_calvin_4_partitions/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_calvin_4_partitions/sequencer/Makefile.inc b/src_calvin_4_partitions/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_calvin_4_partitions/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_4_partitions/sequencer/sequencer.h b/src_calvin_4_partitions/sequencer/sequencer.h deleted file mode 100755 index 0e64d97f..00000000 --- a/src_calvin_4_partitions/sequencer/sequencer.h +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 56 - -#define SAMPLES 100000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST - -using std::set; -using std::string; -using std::queue; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client, - Storage* storage); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Pointer to this node's storage object, for prefetching. - Storage* storage_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_calvin_4_partitions/tests/configuration_test.cc b/src_calvin_4_partitions/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_calvin_4_partitions/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_calvin_4_partitions/tests/connection_test.cc b/src_calvin_4_partitions/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_calvin_4_partitions/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_calvin_4_partitions/tests/fetching_storage_test.cc b/src_calvin_4_partitions/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_calvin_4_partitions/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_calvin_4_partitions/tests/paxos_test.cc b/src_calvin_4_partitions/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_calvin_4_partitions/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_calvin_4_partitions/tests/profile_test.cc b/src_calvin_4_partitions/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_calvin_4_partitions/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_calvin_4_partitions/tests/sequencer_test.cc b/src_calvin_4_partitions/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_calvin_4_partitions/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_calvin_4_partitions/tests/serial_scheduler_test.cc b/src_calvin_4_partitions/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_calvin_4_partitions/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_calvin_4_partitions/tests/storage_manager_test.cc b/src_calvin_4_partitions/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_calvin_4_partitions/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_calvin_4_partitions/tests/tpcc_test.cc b/src_calvin_4_partitions/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_calvin_4_partitions/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_calvin_4_partitions/tests/types_test.cc b/src_calvin_4_partitions/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_calvin_4_partitions/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_calvin_4_partitions/Makefile b/src_calvin_ext/Makefile old mode 100755 new mode 100644 similarity index 97% rename from src_calvin_4_partitions/Makefile rename to src_calvin_ext/Makefile index e3e6e17d..c6b06741 --- a/src_calvin_4_partitions/Makefile +++ b/src_calvin_ext/Makefile @@ -47,7 +47,7 @@ CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ -I$(PROTOB)/src \ -I$(GTEST)/include \ -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror +CXXFLAGS += -Wall -Werror -Wno-format-truncation -Wno-unused-result -Wno-maybe-uninitialized LDFLAGS := -lrt -lpthread $(PG) \ -lprotobuf -L$(PROTOB)/src/.libs \ diff --git a/src_calvin_3_partitions/Makefile.template b/src_calvin_ext/Makefile.template old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/Makefile.template rename to src_calvin_ext/Makefile.template diff --git a/src_dependent_remote_index/applications/Makefile.inc b/src_calvin_ext/applications/Makefile.inc old mode 100755 new mode 100644 similarity index 94% rename from src_dependent_remote_index/applications/Makefile.inc rename to src_calvin_ext/applications/Makefile.inc index db42fd1b..7bd88aa8 --- a/src_dependent_remote_index/applications/Makefile.inc +++ b/src_calvin_ext/applications/Makefile.inc @@ -4,7 +4,7 @@ OBJDIRS += applications UPPERC_DIR := APPLICATIONS LOWERC_DIR := applications -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc +APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc applications/ycsb.cc SRC_LINKED_OBJECTS := $(PROTO_OBJS) TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) diff --git a/src_calvin_vector_vll/applications/application.h b/src_calvin_ext/applications/application.h similarity index 89% rename from src_calvin_vector_vll/applications/application.h rename to src_calvin_ext/applications/application.h index 1b62b033..f62e1353 100644 --- a/src_calvin_vector_vll/applications/application.h +++ b/src_calvin_ext/applications/application.h @@ -30,7 +30,9 @@ class Application { virtual ~Application() {} // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, + virtual TxnProto* NewTxn(int64 txn_id, + int txn_type, + string args, Configuration* config) const = 0; // Static method to convert a key into an int for an array diff --git a/src_calvin_vector_vll/applications/microbenchmark.cc b/src_calvin_ext/applications/microbenchmark.cc similarity index 67% rename from src_calvin_vector_vll/applications/microbenchmark.cc rename to src_calvin_ext/applications/microbenchmark.cc index 93dddedc..4185d1b7 100644 --- a/src_calvin_vector_vll/applications/microbenchmark.cc +++ b/src_calvin_ext/applications/microbenchmark.cc @@ -11,16 +11,20 @@ #include "backend/storage_manager.h" #include "common/utils.h" #include "common/configuration.h" +#include "common/definitions.hh" #include "proto/txn.pb.h" // #define PREFETCHING -#define COLD_CUTOFF 990000 // Fills '*keys' with num_keys unique ints k where // 'key_start' <= k < 'key_limit', and k == part (mod nparts). // Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { +void Microbenchmark::GetRandomKeys(set* keys, + int num_keys, + int key_start, + int key_limit, + int part, + bool is_uniform) { assert(key_start % nparts == 0); keys->clear(); for (int i = 0; i < num_keys; i++) { @@ -28,7 +32,8 @@ void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, int key; do { key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); + nparts * (((is_uniform) ? rand() : zipf_()) % + ((key_limit - key_start) / nparts)); } while (keys->count(key)); keys->insert(key); } @@ -43,7 +48,7 @@ TxnProto* Microbenchmark::InitializeTxn() { txn->set_txn_type(INITIALIZE); // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) + for (int i = 0; i < DB_SIZE; i++) txn->add_write_set(IntToString(i)); return txn; @@ -58,18 +63,17 @@ TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { txn->set_txn_id(txn_id); txn->set_txn_type(MICROTXN_SP); - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); + // // Add one hot key to read/write set. + // int hotkey = part + nparts * (rand() % hot_records); + // txn->add_read_write_set(IntToString(hotkey)); - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. + bool is_uniform = (rand() % 100) < UNIFORM_KEY_SELECTION_RATIO; + + // Insert set of RW_SET_SIZE - 1 random cold keys from specified partition + // into read/write set. set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); + GetRandomKeys(&keys, RW_SET_SIZE, nparts * hot_records, nparts * DB_SIZE, + part, is_uniform); // uniform dist for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); @@ -92,21 +96,17 @@ TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { txn->add_read_write_set(IntToString(hotkey1)); txn->add_read_write_set(IntToString(hotkey2)); - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into + bool is_uniform = (rand() % 100) < UNIFORM_KEY_SELECTION_RATIO; + + // Insert set of RW_SET_SIZE/2 - 1 random cold keys from each partition into // read/write set. set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); + GetRandomKeys(&keys, RW_SET_SIZE / 2 - 1, nparts * hot_records, + nparts * DB_SIZE, part1, is_uniform); for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); + GetRandomKeys(&keys, RW_SET_SIZE / 2 - 1, nparts * hot_records, + nparts * DB_SIZE, part2, is_uniform); for (set::iterator it = keys.begin(); it != keys.end(); ++it) txn->add_read_write_set(IntToString(*it)); @@ -115,8 +115,10 @@ TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { // The load generator can be called externally to return a transaction proto // containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { +TxnProto* Microbenchmark::NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration* config) const { return NULL; } @@ -124,31 +126,29 @@ int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { // Read all elements of 'txn->read_set()', add one to each, write them all // back out. - for (int i = 0; i < kRWSetSize; i++) { + for (int i = 0; i < RW_SET_SIZE; i++) { Value* val = storage->ReadObject(txn->read_write_set(i)); *val = IntToString(StringToInt(*val) + 1); // Not necessary since storage already has a pointer to val. // storage->PutObject(txn->read_write_set(i), val); - // The following code is for microbenchmark "long" transaction, uncomment it if for "long" transaction + // The following code is for microbenchmark "long" transaction, uncomment it + // if for "long" transaction /**int x = 1; for(int i = 0; i < 1100; i++) { x = x*x+1; x = x+10; x = x-2; }**/ - } return 0; } void Microbenchmark::InitializeStorage(Storage* storage, Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { + for (int i = 0; i < nparts * DB_SIZE; i++) { if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { storage->PutObject(IntToString(i), new Value(IntToString(i))); - } } } - diff --git a/src_calvin_vector_vll/applications/microbenchmark.h b/src_calvin_ext/applications/microbenchmark.h similarity index 64% rename from src_calvin_vector_vll/applications/microbenchmark.h rename to src_calvin_ext/applications/microbenchmark.h index a6dd675a..5c859ab2 100644 --- a/src_calvin_vector_vll/applications/microbenchmark.h +++ b/src_calvin_ext/applications/microbenchmark.h @@ -11,26 +11,35 @@ #include #include "applications/application.h" +#include "common/definitions.hh" +#include "common/random.hh" +#include "common/zipf.hh" using std::set; using std::string; class Microbenchmark : public Application { public: + Xoroshiro128Plus rnd_; + FastZipf zipf_; + enum TxnType { INITIALIZE = 0, MICROTXN_SP = 1, MICROTXN_MP = 2, }; - Microbenchmark(int nodecount, int hotcount) { + Microbenchmark(int nodecount, int hotcount) + : rnd_(), zipf_(&rnd_, SKEW, DB_SIZE) { nparts = nodecount; hot_records = hotcount; } virtual ~Microbenchmark() {} - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, + virtual TxnProto* NewTxn(int64 txn_id, + int txn_type, + string args, Configuration* config = NULL) const; virtual int Execute(TxnProto* txn, StorageManager* storage) const; @@ -40,16 +49,17 @@ class Microbenchmark : public Application { int nparts; int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} + void GetRandomKeys(set* keys, + int num_keys, + int key_start, + int key_limit, + int part, + bool is_uniform); + Microbenchmark() : rnd_(), zipf_(&rnd_, SKEW, DB_SIZE) {} }; #endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_calvin_ext/applications/tpcc.cc b/src_calvin_ext/applications/tpcc.cc new file mode 100644 index 00000000..07bca5b6 --- /dev/null +++ b/src_calvin_ext/applications/tpcc.cc @@ -0,0 +1,1198 @@ +// Author: Kun Ren (kun.ren@yale.edu) +// Author: Thaddeus Diamond (diamond@cs.yale.edu) +// +// +// A concrete implementation of TPC-C (application subclass) + +#include "applications/tpcc.h" + +#include +#include + +#include "backend/storage.h" +#include "backend/storage_manager.h" +#include "common/configuration.h" +#include "common/utils.h" +#include "proto/tpcc.pb.h" +#include "proto/tpcc_args.pb.h" + +using std::string; + +// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // +unordered_map ItemList; +Value *TPCC::GetItem(Key key) const +{ + return ItemList[key]; +} +void TPCC::SetItem(Key key, Value *value) const +{ + ItemList[key] = value; +} + +// The load generator can be called externally to return a +// transaction proto containing a new type of transaction. +TxnProto *TPCC::NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration *config) const +{ + // Create the new transaction object + TxnProto *txn = new TxnProto(); + + // Set the transaction's standard attributes + txn->set_txn_id(txn_id); + txn->set_txn_type(txn_type); + txn->set_isolation_level(TxnProto::SERIALIZABLE); + txn->set_status(TxnProto::NEW); + txn->set_multipartition(false); + + // Parse out the arguments to the transaction + TPCCArgs *txn_args = new TPCCArgs(); + assert(txn_args->ParseFromString(args)); + bool mp = txn_args->multipartition(); + int remote_node; + if (mp) + { + do + { + remote_node = rand() % config->all_nodes.size(); + } while (config->all_nodes.size() > 1 && + remote_node == config->this_node_id); + } + + // Create an arg list + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->set_system_time(GetTime()); + + // Because a switch is not scoped we declare our variables outside of it + int warehouse_id, district_id, customer_id; + char warehouse_key[128], district_key[128], customer_key[128]; + int order_line_count; + bool invalid; + Value customer_value; + std::set items_used; + + // We set the read and write set based on type + switch (txn_type) + { + // Initialize + case INITIALIZE: + // Finished with INITIALIZE txn creation + break; + + // New Order + case NEW_ORDER: + // First, we pick a local warehouse + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); + + txn->add_read_set(warehouse_key); + + // Next, we pick a random district + district_id = rand() % DISTRICTS_PER_WAREHOUSE; + snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, + district_id); + txn->add_read_write_set(district_key); + + // Finally, we pick a random customer + customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", warehouse_id, + district_id, customer_id); + txn->add_read_set(customer_key); + + int order_number; + if (next_order_id_for_district.count(district_key) > 0) + { + order_number = next_order_id_for_district[district_key]; + next_order_id_for_district[district_key]++; + } + else + { + order_number = 0; + next_order_id_for_district[district_key] = 1; + } + + // We set the length of the read and write set uniformly between 5 and 15 + order_line_count = (rand() % 11) + 5; + + // Let's choose a bad transaction 1% of the time + invalid = false; + // if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) + // invalid = true; + + // Iterate through each order line + for (int i = 0; i < order_line_count; i++) + { + // Set the item id (Invalid orders have the last item be -1) + int item; + do + { + item = rand() % NUMBER_OF_ITEMS; + } while (items_used.count(item) > 0); + items_used.insert(item); + + if (invalid && i == order_line_count - 1) + item = -1; + + // Print the item key into a buffer + char item_key[128]; + snprintf(item_key, sizeof(item_key), "i%d", item); + + // Create an order line warehouse key (default is local) + char remote_warehouse_key[128]; + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "%s", + warehouse_key); + + // We only do ~1% remote transactions + if (mp) + { + txn->set_multipartition(true); + + // We loop until we actually get a remote one + int remote_warehouse_id; + do + { + remote_warehouse_id = + rand() % (WAREHOUSES_PER_NODE * config->all_nodes.size()); + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", + remote_warehouse_id); + } while (config->all_nodes.size() > 1 && + config->LookupPartition(remote_warehouse_key) != + remote_node); + } + + // Determine if we should add it to read set to avoid duplicates + bool needed = true; + for (int j = 0; j < txn->read_set_size(); j++) + { + if (txn->read_set(j) == remote_warehouse_key) + needed = false; + } + if (needed) + txn->add_read_set(remote_warehouse_key); + + // Finally, we set the stock key to the read and write set + Key stock_key = string(remote_warehouse_key) + "s" + item_key; + txn->add_read_write_set(stock_key); + + // Set the quantity randomly within [1..10] + tpcc_args->add_quantities(rand() % 10 + 1); + + // Finally, we add the order line key to the write set + char order_line_key[128]; + snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", + district_key, order_number, i); + txn->add_write_set(order_line_key); + } + + // Create a new order key to add to write set + char new_order_key[128]; + snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, + order_number); + txn->add_write_set(new_order_key); + + // Create an order key to add to write set + char order_key[128]; + snprintf(order_key, sizeof(order_key), "%so%d", district_key, + order_number); + txn->add_write_set(order_key); + + // Set the order line count in the args + tpcc_args->add_order_line_count(order_line_count); + tpcc_args->set_order_number(order_number); + break; + + // Payment + case PAYMENT: + // Specify an amount for the payment + tpcc_args->set_amount( + rand() / (static_cast(RAND_MAX + 1.0)) * 4999.0 + 1); + + // First, we pick a local warehouse + + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", warehouse_id); + txn->add_read_write_set(warehouse_key); + + // Next, we pick a district + district_id = rand() % DISTRICTS_PER_WAREHOUSE; + snprintf(district_key, sizeof(district_key), "w%dd%dy", warehouse_id, + district_id); + txn->add_read_write_set(district_key); + + // Add history key to write set + char history_key[128]; + snprintf(history_key, sizeof(history_key), "w%dh%ld", warehouse_id, + txn->txn_id()); + txn->add_write_set(history_key); + + // Next, we find the customer as a local one + if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) + { + customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", warehouse_id, + district_id, customer_id); + + // If the probability is 15%, we make it a remote customer + } + else + { + int remote_warehouse_id; + int remote_district_id; + int remote_customer_id; + char remote_warehouse_key[40]; + do + { + remote_warehouse_id = + rand() % (WAREHOUSES_PER_NODE * config->all_nodes.size()); + snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", + remote_warehouse_id); + + remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; + + remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", + remote_warehouse_id, remote_district_id, remote_customer_id); + } while (config->all_nodes.size() > 1 && + config->LookupPartition(remote_warehouse_key) != remote_node); + } + + // We only do secondary keying ~60% of the time + if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) + { + // Now that we have the object, let's create the txn arg + tpcc_args->set_last_name(customer_key); + txn->add_read_set(customer_key); + + // Otherwise just give a customer key + } + else + { + txn->add_read_write_set(customer_key); + } + + break; + + case ORDER_STATUS: + { + string customer_string; + string customer_latest_order; + string warehouse_string; + string district_string; + int customer_order_line_number; + + if (latest_order_id_for_customer.size() < 1) + { + txn->set_txn_id(-1); + break; + } + + pthread_mutex_lock(&mutex_); + customer_string = + (*involed_customers)[rand() % involed_customers->size()]; + pthread_mutex_unlock(&mutex_); + customer_latest_order = latest_order_id_for_customer[customer_string]; + warehouse_string = customer_string.substr(0, customer_string.find("d")); + district_string = customer_string.substr(0, customer_string.find("c")); + snprintf(customer_key, sizeof(customer_key), "%s", + customer_string.c_str()); + snprintf(warehouse_key, sizeof(warehouse_key), "%s", + warehouse_string.c_str()); + snprintf(district_key, sizeof(district_key), "%s", + district_string.c_str()); + + customer_order_line_number = order_line_number[customer_latest_order]; + txn->add_read_set(warehouse_key); + txn->add_read_set(district_key); + txn->add_read_set(customer_key); + + snprintf(order_key, sizeof(order_key), "%s", + customer_latest_order.c_str()); + txn->add_read_set(order_key); + char order_line_key[128]; + for (int i = 0; i < customer_order_line_number; i++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + i); + txn->add_read_set(order_line_key); + } + + tpcc_args->add_order_line_count(customer_order_line_number); + + break; + } + + case STOCK_LEVEL: + { + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); + + // Next, we pick a random district + district_id = rand() % DISTRICTS_PER_WAREHOUSE; + snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, + district_id); + + if (latest_order_id_for_district.count(district_key) == 0) + { + txn->set_txn_id(-1); + break; + } + + txn->add_read_set(warehouse_key); + txn->add_read_set(district_key); + int latest_order_number = latest_order_id_for_district[district_key]; + char order_line_key[128]; + char stock_key[128]; + + tpcc_args->set_lastest_order_number(latest_order_number); + tpcc_args->set_threshold(rand() % 10 + 10); + + for (int i = latest_order_number; + (i >= 0) && (i > latest_order_number - 20); i--) + { + snprintf(order_key, sizeof(order_key), "%so%d", district_key, i); + int ol_number = order_line_number[order_key]; + + for (int j = 0; j < ol_number; j++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + j); + int item = item_for_order_line[order_line_key]; + if (items_used.count(item) > 0) + { + continue; + } + items_used.insert(item); + txn->add_read_set(order_line_key); + snprintf(stock_key, sizeof(stock_key), "%ssi%d", warehouse_key, item); + txn->add_read_set(stock_key); + } + } + + break; + } + + case DELIVERY: + { + warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + + config->this_node_id; + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); + txn->add_read_set(warehouse_key); + + char order_line_key[128]; + int oldest_order; + + for (int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) + { + snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); + if ((smallest_order_id_for_district.count(district_key) == 0) || + (smallest_order_id_for_district[district_key] > + latest_order_id_for_district[district_key])) + { + continue; + } + else + { + txn->add_read_set(district_key); + oldest_order = smallest_order_id_for_district[district_key]; + smallest_order_id_for_district[district_key]++; + + snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, + oldest_order); + txn->add_read_write_set(new_order_key); + } + + snprintf(order_key, sizeof(order_key), "%so%d", district_key, + oldest_order); + txn->add_read_write_set(order_key); + int ol_number = order_line_number[order_key]; + tpcc_args->add_order_line_count(ol_number); + + for (int j = 0; j < ol_number; j++) + { + snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, + j); + txn->add_read_write_set(order_line_key); + } + + snprintf(customer_key, sizeof(customer_key), "%s", + customer_for_order[order_key].c_str()); + txn->add_read_write_set(customer_key); + } + + break; + } + + // Invalid transaction + default: + break; + } + + // Set the transaction's args field to a serialized version + Value args_string; + assert(tpcc_args->SerializeToString(&args_string)); + txn->set_arg(args_string); + + // Free memory + delete tpcc_args; + delete txn_args; + + return txn; +} + +// The execute function takes a single transaction proto and executes it based +// on what the type of the transaction is. +int TPCC::Execute(TxnProto *txn, StorageManager *storage) const +{ + switch (txn->txn_type()) + { + // Initialize + case INITIALIZE: + InitializeStorage(storage->GetStorage(), NULL); + return SUCCESS; + break; + + // New Order + case NEW_ORDER: + return NewOrderTransaction(txn, storage); + break; + + // Payment + case PAYMENT: + return PaymentTransaction(txn, storage); + break; + + case ORDER_STATUS: + return OrderStatusTransaction(txn, storage); + break; + + case STOCK_LEVEL: + return StockLevelTransaction(txn, storage); + break; + + case DELIVERY: + return DeliveryTransaction(txn, storage); + break; + + // Invalid transaction + default: + return FAILURE; + break; + } + + return FAILURE; +} + +// The new order function is executed when the application receives a new order +// transaction. This follows the TPC-C standard. +int TPCC::NewOrderTransaction(TxnProto *txn, StorageManager *storage) const +{ + // First, we retrieve the warehouse from storage + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + // Next, we retrieve the district + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_write_set(0)); + assert(district->ParseFromString(*district_value)); + // Increment the district's next order ID and put to datastore + district->set_next_order_id(district->next_order_id() + 1); + assert(district->SerializeToString(district_value)); + // Not necessary since storage already has a pointer to district_value. + // storage->PutObject(district->id(), district_value); + + // Retrieve the customer we are looking for + Value *customer_value; + Customer *customer = new Customer(); + customer_value = storage->ReadObject(txn->read_set(1)); + assert(customer->ParseFromString(*customer_value)); + + // Next, we get the order line count, system time, and other args from the + // transaction proto + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int order_line_count = tpcc_args->order_line_count(0); + int order_number = tpcc_args->order_number(); + double system_time = tpcc_args->system_time(); + + // Next we create an Order object + Key order_key = txn->write_set(order_line_count + 1); + Order *order = new Order(); + order->set_id(order_key); + order->set_warehouse_id(warehouse->id()); + order->set_district_id(district->id()); + order->set_customer_id(customer->id()); + + // Set some of the auxiliary data + order->set_entry_date(system_time); + order->set_carrier_id(-1); + order->set_order_line_count(order_line_count); + order->set_all_items_local(txn->multipartition()); + + // We initialize the order line amount total to 0 + int order_line_amount_total = 0; + + for (int i = 0; i < order_line_count; i++) + { + // For each order line we parse out the three args + + string stock_key = txn->read_write_set(i + 1); + string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); + int quantity = tpcc_args->quantities(i); + + // Find the item key within the stock key + size_t item_idx = stock_key.find("i"); + string item_key = stock_key.substr(item_idx, string::npos); + + // First, we check if the item number is valid + Item *item = new Item(); + if (item_key == "i-1") + return FAILURE; + else + assert(item->ParseFromString(*GetItem(item_key))); + + // Next, we create a new order line object with std attributes + OrderLine *order_line = new OrderLine(); + Key order_line_key = txn->write_set(i); + order_line->set_order_id(order_line_key); + + // Set the attributes for this order line + order_line->set_district_id(district->id()); + order_line->set_warehouse_id(warehouse->id()); + order_line->set_number(i); + order_line->set_item_id(item_key); + order_line->set_supply_warehouse_id(supply_warehouse_key); + order_line->set_quantity(quantity); + order_line->set_delivery_date(system_time); + + // Next, we get the correct stock from the data store + Stock *stock = new Stock(); + Value *stock_value = storage->ReadObject(stock_key); + assert(stock->ParseFromString(*stock_value)); + + // Once we have it we can increase the YTD, order_count, and remote_count + stock->set_year_to_date(stock->year_to_date() + quantity); + stock->set_order_count(stock->order_count() - 1); + if (txn->multipartition()) + stock->set_remote_count(stock->remote_count() + 1); + + // And we decrease the stock's supply appropriately and rewrite to storage + if (stock->quantity() >= quantity + 10) + stock->set_quantity(stock->quantity() - quantity); + else + stock->set_quantity(stock->quantity() - quantity + 91); + + // Put the stock back into the database + assert(stock->SerializeToString(stock_value)); + // Not necessary since storage already has a ptr to stock_value. + // storage->PutObject(stock_key, stock_value); + delete stock; + + // Next, we update the order line's amount and add it to the running sum + order_line->set_amount(quantity * item->price()); + order_line_amount_total += (quantity * item->price()); + + // Finally, we write the order line to storage + Value *order_line_value = new Value(); + assert(order_line->SerializeToString(order_line_value)); + storage->PutObject(order_line_key, order_line_value); + // order->add_order_line_ptr(reinterpret_cast(order_line_value)); + + pthread_mutex_lock(&mutex_for_item); + if (storage->configuration_->this_node_id == + storage->configuration_->LookupPartition(txn->read_set(0))) + item_for_order_line[order_line_key] = StringToInt(item_key); + pthread_mutex_unlock(&mutex_for_item); + // Free memory + delete order_line; + delete item; + } + + // We create a new NewOrder object + Key new_order_key = txn->write_set(order_line_count); + NewOrder *new_order = new NewOrder(); + new_order->set_id(new_order_key); + new_order->set_warehouse_id(warehouse->id()); + new_order->set_district_id(district->id()); + + // Serialize it and put it in the datastore + Value *new_order_value = new Value(); + assert(new_order->SerializeToString(new_order_value)); + storage->PutObject(new_order_key, new_order_value); + + // Serialize order and put it in the datastore + Value *order_value = new Value(); + assert(order->SerializeToString(order_value)); + storage->PutObject(order_key, order_value); + + if (storage->configuration_->this_node_id == + storage->configuration_->LookupPartition(txn->read_set(0))) + { + pthread_mutex_lock(&mutex_); + if (latest_order_id_for_customer.count(txn->read_set(1)) == 0) + involed_customers->push_back(txn->read_set(1)); + latest_order_id_for_customer[txn->read_set(1)] = order_key; + + if (smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) + { + smallest_order_id_for_district[txn->read_write_set(0)] = order_number; + } + order_line_number[order_key] = order_line_count; + customer_for_order[order_key] = txn->read_set(1); + + latest_order_id_for_district[txn->read_write_set(0)] = order_number; + pthread_mutex_unlock(&mutex_); + } + + // Successfully completed transaction + delete warehouse; + delete district; + delete customer; + delete order; + delete new_order; + delete tpcc_args; + return SUCCESS; +} + +// The payment function is executed when the application receives a +// payment transaction. This follows the TPC-C standard. +int TPCC::PaymentTransaction(TxnProto *txn, StorageManager *storage) const +{ + // First, we parse out the transaction args from the TPCC proto + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int amount = tpcc_args->amount(); + + // We create a string to hold up the customer object we look up + Value *customer_value; + Key customer_key; + + // If there's a last name we do secondary keying + if (tpcc_args->has_last_name()) + { + Key secondary_key = tpcc_args->last_name(); + + // If the RW set is not at least of size 3, then no customer key was + // given to this transaction. Otherwise, we perform a check to ensure + // the secondary key we just looked up agrees with our previous lookup + if (txn->read_write_set_size() < 3 || + secondary_key != txn->read_write_set(2)) + { + // Append the newly read key to write set + if (txn->read_write_set_size() < 3) + txn->add_read_write_set(secondary_key); + + // Or the old one was incorrect so we overwrite it + else + txn->set_read_write_set(2, secondary_key); + + return REDO; + // Otherwise, we look up the customer's key + } + else + { + customer_value = storage->ReadObject(tpcc_args->last_name()); + } + + // Otherwise we use the final argument + } + else + { + customer_key = txn->read_write_set(2); + customer_value = storage->ReadObject(customer_key); + } + + // Deserialize the warehouse object + Key warehouse_key = txn->read_write_set(0); + Value *warehouse_value = storage->ReadObject(warehouse_key); + Warehouse *warehouse = new Warehouse(); + assert(warehouse->ParseFromString(*warehouse_value)); + + // Next, we update the values of the warehouse and write it out + warehouse->set_year_to_date(warehouse->year_to_date() + amount); + assert(warehouse->SerializeToString(warehouse_value)); + // Not necessary since storage already has a pointer to warehouse_value. + // storage->PutObject(warehouse_key, warehouse_value); + + // Deserialize the district object + Key district_key = txn->read_write_set(1); + Value *district_value = storage->ReadObject(district_key); + District *district = new District(); + assert(district->ParseFromString(*district_value)); + + // Next, we update the values of the district and write it out + district->set_year_to_date(district->year_to_date() + amount); + assert(district->SerializeToString(district_value)); + // Not necessary since storage already has a pointer to district_value. + // storage->PutObject(district_key, district_value); + + // We deserialize the customer + Customer *customer = new Customer(); + assert(customer->ParseFromString(*customer_value)); + + // Next, we update the customer's balance, payment and payment count + customer->set_balance(customer->balance() - amount); + customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); + customer->set_payment_count(customer->payment_count() + 1); + + // If the customer has bad credit, we update the data information attached + // to her + if (customer->credit() == "BC") + { + string data = customer->data(); + char new_information[500]; + + // Print the new_information into the buffer + snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", + customer->id().c_str(), customer->warehouse_id().c_str(), + customer->district_id().c_str(), district->id().c_str(), + warehouse->id().c_str(), amount, customer->data().c_str()); + } + + // We write the customer to disk + assert(customer->SerializeToString(customer_value)); + // Not necessary since storage already has a pointer to customer_value. + // storage->PutObject(customer_key, customer_value); + + // Finally, we create a history object and update the data + History *history = new History(); + history->set_customer_id(customer_key); + history->set_customer_warehouse_id(customer->warehouse_id()); + history->set_customer_district_id(customer->district_id()); + history->set_warehouse_id(warehouse_key); + history->set_district_id(district_key); + + // Create the data for the history object + char history_data[100]; + snprintf(history_data, sizeof(history_data), "%s %s", + warehouse->name().c_str(), district->name().c_str()); + history->set_data(history_data); + + // Write the history object to disk + Value *history_value = new Value(); + assert(history->SerializeToString(history_value)); + storage->PutObject(txn->write_set(0), history_value); + + // Successfully completed transaction + delete customer; + delete history; + delete district; + delete warehouse; + delete tpcc_args; + return SUCCESS; +} + +int TPCC::OrderStatusTransaction(TxnProto *txn, StorageManager *storage) const +{ + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int order_line_count = tpcc_args->order_line_count(0); + + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(1)); + assert(district->ParseFromString(*district_value)); + + Value *customer_value; + Customer *customer = new Customer(); + customer_value = storage->ReadObject(txn->read_set(2)); + assert(customer->ParseFromString(*customer_value)); + + // double customer_balance = customer->balance(); + string customer_first = customer->first(); + string customer_middle = customer->middle(); + string customer_last = customer->last(); + + Order *order = new Order(); + Value *order_value = storage->ReadObject(txn->read_set(3)); + assert(order->ParseFromString(*order_value)); + // int carrier_id = order->carrier_id(); + // double entry_date = order->entry_date(); + + for (int i = 0; i < order_line_count; i++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = storage->ReadObject(txn->read_set(4 + i)); + assert(order_line->ParseFromString(*order_line_value)); + string item_key = order_line->item_id(); + string supply_warehouse_id = order_line->supply_warehouse_id(); + // int quantity = order_line->quantity(); + // double amount = order_line->amount(); + // double delivery_date = order_line->delivery_date(); + + delete order_line; + } + + delete warehouse; + delete district; + delete customer; + delete order; + + return SUCCESS; +} + +int TPCC::StockLevelTransaction(TxnProto *txn, StorageManager *storage) const +{ + int low_stock = 0; + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + int threshold = tpcc_args->threshold(); + + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(1)); + assert(district->ParseFromString(*district_value)); + + int index = 0; + + int cycle = (txn->read_set_size() - 2) / 2; + for (int i = 0; i < cycle; i++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = storage->ReadObject(txn->read_set(2 + index)); + index++; + assert(order_line->ParseFromString(*order_line_value)); + string item_key = order_line->item_id(); + + Stock *stock = new Stock(); + Value *stock_value = storage->ReadObject(txn->read_set(2 + index)); + index++; + assert(stock->ParseFromString(*stock_value)); + if (stock->quantity() < threshold) + { + low_stock++; + } + + delete order_line; + delete stock; + } + + delete warehouse; + delete district; + return SUCCESS; +} + +int TPCC::DeliveryTransaction(TxnProto *txn, StorageManager *storage) const +{ + TPCCArgs *tpcc_args = new TPCCArgs(); + tpcc_args->ParseFromString(txn->arg()); + + Value *warehouse_value; + Warehouse *warehouse = new Warehouse(); + warehouse_value = storage->ReadObject(txn->read_set(0)); + assert(warehouse->ParseFromString(*warehouse_value)); + + if (txn->read_set_size() == 1) + { + delete warehouse; + return SUCCESS; + } + + int delivery_district_number = txn->read_set_size() - 1; + int read_write_index = 0; + int line_count_index = 0; + for (int i = 1; i <= delivery_district_number; i++) + { + District *district = new District(); + Value *district_value = storage->ReadObject(txn->read_set(i)); + assert(district->ParseFromString(*district_value)); + + storage->DeleteObject(txn->read_write_set(read_write_index)); + read_write_index++; + + Order *order = new Order(); + Value *order_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(order->ParseFromString(*order_value)); + char customer_key[128]; + snprintf(customer_key, sizeof(customer_key), "%s", + order->customer_id().c_str()); + + order->set_carrier_id(rand() % 10); + + int ol_number = tpcc_args->order_line_count(line_count_index); + line_count_index++; + double total_amount = 0; + + for (int j = 0; j < ol_number; j++) + { + OrderLine *order_line = new OrderLine(); + Value *order_line_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(order_line->ParseFromString(*order_line_value)); + order_line->set_delivery_date(GetTime()); + total_amount = total_amount + order_line->amount(); + + delete order_line; + } + + Customer *customer = new Customer(); + Value *customer_value = + storage->ReadObject(txn->read_write_set(read_write_index)); + read_write_index++; + assert(customer->ParseFromString(*customer_value)); + customer->set_balance(customer->balance() + total_amount); + customer->set_delivery_count(customer->delivery_count() + 1); + + delete district; + delete order; + delete customer; + } + + delete warehouse; + return SUCCESS; +} + +// The initialize function is executed when an initialize transaction comes +// through, indicating we should populate the database with fake data +void TPCC::InitializeStorage(Storage *storage, Configuration *conf) const +{ + // We create and write out all of the warehouses + for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); + i++) + { + // First, we create a key for the warehouse + char warehouse_key[128], warehouse_key_ytd[128]; + Value *warehouse_value = new Value(); + snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); + snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); + if (conf->LookupPartition(warehouse_key) != conf->this_node_id) + { + continue; + } + // Next we initialize the object and serialize it + Warehouse *warehouse = CreateWarehouse(warehouse_key); + assert(warehouse->SerializeToString(warehouse_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(warehouse_key) == conf->this_node_id) + { + storage->PutObject(warehouse_key, warehouse_value); + storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); + } + + // Next, we create and write out all of the districts + for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) + { + // First, we create a key for the district + char district_key[128], district_key_ytd[128]; + snprintf(district_key, sizeof(district_key), "w%dd%d", i, j); + snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", i, j); + + // Next we initialize the object and serialize it + Value *district_value = new Value(); + District *district = CreateDistrict(district_key, warehouse_key); + assert(district->SerializeToString(district_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(district_key) == conf->this_node_id) + { + storage->PutObject(district_key, district_value); + storage->PutObject(district_key_ytd, new Value(*district_value)); + } + + // Next, we create and write out all of the customers + for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) + { + // First, we create a key for the customer + char customer_key[128]; + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", i, j, k); + + // Next we initialize the object and serialize it + Value *customer_value = new Value(); + Customer *customer = + CreateCustomer(customer_key, district_key, warehouse_key); + assert(customer->SerializeToString(customer_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(customer_key) == conf->this_node_id) + storage->PutObject(customer_key, customer_value); + delete customer; + } + + // Free storage + delete district; + } + + // Next, we create and write out all of the stock + for (int j = 0; j < NUMBER_OF_ITEMS; j++) + { + // First, we create a key for the stock + char item_key[128]; + Value *stock_value = new Value(); + snprintf(item_key, sizeof(item_key), "i%d", j); + + // Next we initialize the object and serialize it + Stock *stock = CreateStock(item_key, warehouse_key); + assert(stock->SerializeToString(stock_value)); + + // Finally, we pass it off to the storage manager to write to disk + if (conf->LookupPartition(stock->id()) == conf->this_node_id) + storage->PutObject(stock->id(), stock_value); + delete stock; + } + + // Free storage + delete warehouse; + } + + // Finally, all the items are initialized + srand(1000); + for (int i = 0; i < NUMBER_OF_ITEMS; i++) + { + // First, we create a key for the item + char item_key[128]; + Value *item_value = new Value(); + snprintf(item_key, sizeof(item_key), "i%d", i); + + // Next we initialize the object and serialize it + Item *item = CreateItem(item_key); + assert(item->SerializeToString(item_value)); + + // Finally, we pass it off to the local record of items + SetItem(string(item_key), item_value); + delete item; + } +} + +// The following method is a dumb constructor for the warehouse protobuffer +Warehouse *TPCC::CreateWarehouse(Key warehouse_key) const +{ + Warehouse *warehouse = new Warehouse(); + + // We initialize the id and the name fields + warehouse->set_id(warehouse_key); + warehouse->set_name(RandomString(10)); + + // Provide some information to make TPC-C happy + warehouse->set_street_1(RandomString(20)); + warehouse->set_street_2(RandomString(20)); + warehouse->set_city(RandomString(20)); + warehouse->set_state(RandomString(2)); + warehouse->set_zip(RandomString(9)); + + // Set default financial information + warehouse->set_tax(0.05); + warehouse->set_year_to_date(0.0); + + return warehouse; +} + +District *TPCC::CreateDistrict(Key district_key, Key warehouse_key) const +{ + District *district = new District(); + + // We initialize the id and the name fields + district->set_id(district_key); + district->set_warehouse_id(warehouse_key); + district->set_name(RandomString(10)); + + // Provide some information to make TPC-C happy + district->set_street_1(RandomString(20)); + district->set_street_2(RandomString(20)); + district->set_city(RandomString(20)); + district->set_state(RandomString(2)); + district->set_zip(RandomString(9)); + + // Set default financial information + district->set_tax(0.05); + district->set_year_to_date(0.0); + district->set_next_order_id(1); + + return district; +} + +Customer *TPCC::CreateCustomer(Key customer_key, + Key district_key, + Key warehouse_key) const +{ + Customer *customer = new Customer(); + + // We initialize the various keys + customer->set_id(customer_key); + customer->set_district_id(district_key); + customer->set_warehouse_id(warehouse_key); + + // Next, we create a first and middle name + customer->set_first(RandomString(20)); + customer->set_middle(RandomString(20)); + customer->set_last(customer_key); + + // Provide some information to make TPC-C happy + customer->set_street_1(RandomString(20)); + customer->set_street_2(RandomString(20)); + customer->set_city(RandomString(20)); + customer->set_state(RandomString(2)); + customer->set_zip(RandomString(9)); + + // Set default financial information + customer->set_since(0); + customer->set_credit("GC"); + customer->set_credit_limit(0.01); + customer->set_discount(0.5); + customer->set_balance(0); + customer->set_year_to_date_payment(0); + customer->set_payment_count(0); + customer->set_delivery_count(0); + + // Set some miscellaneous data + customer->set_data(RandomString(50)); + + return customer; +} + +Stock *TPCC::CreateStock(Key item_key, Key warehouse_key) const +{ + Stock *stock = new Stock(); + + // We initialize the various keys + char stock_key[128]; + snprintf(stock_key, sizeof(stock_key), "%ss%s", warehouse_key.c_str(), + item_key.c_str()); + stock->set_id(stock_key); + stock->set_warehouse_id(warehouse_key); + stock->set_item_id(item_key); + + // Next, we create a first and middle name + stock->set_quantity(rand() % 100 + 100); + + // Set default financial information + stock->set_year_to_date(0); + stock->set_order_count(0); + stock->set_remote_count(0); + + // Set some miscellaneous data + stock->set_data(RandomString(50)); + + return stock; +} + +Item *TPCC::CreateItem(Key item_key) const +{ + Item *item = new Item(); + + // We initialize the item's key + item->set_id(item_key); + + // Initialize some fake data for the name, price and data + item->set_name(RandomString(24)); + item->set_price(rand() % 100); + item->set_data(RandomString(50)); + + return item; +} \ No newline at end of file diff --git a/src_calvin_ext/applications/tpcc.h b/src_calvin_ext/applications/tpcc.h new file mode 100644 index 00000000..6e63d34b --- /dev/null +++ b/src_calvin_ext/applications/tpcc.h @@ -0,0 +1,150 @@ +// Author: Kun Ren (kun.ren@yale.edu) +// Author: Thaddeus Diamond (diamond@cs.yale.edu) +// +// A concrete implementation of TPC-C (application subclass) + +#ifndef _DB_APPLICATIONS_TPCC_H_ +#define _DB_APPLICATIONS_TPCC_H_ + +#include + +#include "applications/application.h" +#include "proto/txn.pb.h" +#include "common/configuration.h" + +// #define WAREHOUSES_PER_NODE 12 +// #define DISTRICTS_PER_WAREHOUSE 10 +#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) +// #define CUSTOMERS_PER_DISTRICT 3000 +#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) +// #define NUMBER_OF_ITEMS 100000 + +#define WAREHOUSES_PER_NODE 1 +#define DISTRICTS_PER_WAREHOUSE 2 +#define CUSTOMERS_PER_DISTRICT 100 +#define NUMBER_OF_ITEMS 1000 +#define ORDERS_PER_DISTRICT 100 + +using std::string; + +class Warehouse; +class District; +class Customer; +class Item; +class Stock; + +class TPCC : public Application +{ +public: + enum TxnType + { + INITIALIZE = 0, + NEW_ORDER = 1, + PAYMENT = 2, + ORDER_STATUS = 3, + DELIVERY = 4, + STOCK_LEVEL = 5, + }; + + virtual ~TPCC() {} + + // Load generator for a new transaction + virtual TxnProto *NewTxn(int64 txn_id, + int txn_type, + string args, + Configuration *config) const; + + // The key converter takes a valid key (string) and converts it to an id + // for the checkpoint to use + static int CheckpointID(Key key) + { + // Initial dissection of the key + size_t id_idx; + + // Switch based on key type + size_t bad = string::npos; + if ((id_idx = key.find("s")) != bad) + { + size_t ware = key.find("w"); + return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("c")) != bad) + { + return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) + { + return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("w")) != bad) + { + return atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("i")) != bad) + { + return 3000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("ol")) != bad) + { + return 4000000 + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("no")) != bad) + { + return 5000000 + atoi(&key[id_idx + 2]); + } + else if ((id_idx = key.find("o")) != bad) + { + return 6000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("h")) != bad) + { + return 7000000 + atoi(&key[id_idx + 1]); + } + else if ((id_idx = key.find("ln")) != bad) + { + return 8000000 + atoi(&key[id_idx + 2]); + } + + // Invalid key + return -1; + } + + // Simple execution of a transaction using a given storage + virtual int Execute(TxnProto *txn, StorageManager *storage) const; + + /* TODO(Thad): Uncomment once testing friend class exists + private: */ + // When the first transaction is called, the following function initializes + // a set of fake data for use in the application + virtual void InitializeStorage(Storage *storage, Configuration *conf) const; + + // The following methods are simple randomized initializers that provide us + // fake data for our TPC-C function + Warehouse *CreateWarehouse(Key id) const; + District *CreateDistrict(Key id, Key warehouse_id) const; + Customer *CreateCustomer(Key id, Key district_id, Key warehouse_id) const; + Item *CreateItem(Key id) const; + Stock *CreateStock(Key id, Key warehouse_id) const; + + // A NewOrder call takes a set of args and a transaction id and performs + // the new order transaction as specified by TPC-C. The return is 1 for + // success or 0 for failure. + int NewOrderTransaction(TxnProto *txn, StorageManager *storage) const; + + // A Payment call takes a set of args as the parameter and performs the + // payment transaction, returning a 1 for success or 0 for failure. + int PaymentTransaction(TxnProto *txn, StorageManager *storage) const; + + int OrderStatusTransaction(TxnProto *txn, StorageManager *storage) const; + + int StockLevelTransaction(TxnProto *txn, StorageManager *storage) const; + + int DeliveryTransaction(TxnProto *txn, StorageManager *storage) const; + + // The following are implementations of retrieval and writing for local items + Value *GetItem(Key key) const; + void SetItem(Key key, Value *value) const; +}; + +#endif // _DB_APPLICATIONS_TPCC_H_ \ No newline at end of file diff --git a/src_calvin_ext/applications/ycsb.cc b/src_calvin_ext/applications/ycsb.cc new file mode 100644 index 00000000..7ffb7b4a --- /dev/null +++ b/src_calvin_ext/applications/ycsb.cc @@ -0,0 +1,283 @@ +// FILE: applications/ycsb.cc + +#include "applications/ycsb.h" + +#include +#include +#include +#include +#include +#include + +#include "backend/storage.h" +#include "backend/storage_manager.h" +#include "common/configuration.h" +#include "common/definitions.hh" +#include "common/random.hh" +#include "common/utils.h" +#include "proto/txn.pb.h" + +enum YCSBTxnType +{ + YCSB_TXN_SP, + YCSB_TXN_MP, + YCSB_TXN_INITIALIZE +}; + +// ---- 補助: Random から [0,1) 一様乱数を作る(Random::Uniform のみで実装)---- +static inline double U01(Random &rnd) +{ + // 2^30 で割ることで [0,1) の double を得る + const uint32_t M = 1u << 30; + return static_cast(rnd.Uniform(M)) / static_cast(M); +} + +// ---- 補助: 簡易 Zipf サンプラー(n が小さいときに使う想定。hot_records_ 用)---- +// 返り値は [0, n-1] +static uint64_t ZipfSample(Random &rnd, uint64_t n, double theta) +{ + if (n <= 1 || theta <= 0.0) + return 0; + + // 定数の前計算(n が小さいので毎回でも軽い) + double zetan = 0.0; + for (uint64_t i = 1; i <= n; ++i) + { + zetan += 1.0 / std::pow(static_cast(i), theta); + } + const double zeta2 = 1.0 + std::pow(2.0, -theta); + const double alpha = 1.0 / (1.0 - theta); + const double eta = (1.0 - std::pow(2.0 / static_cast(n), 1.0 - theta)) / + (1.0 - zeta2 / zetan); + + const double u = U01(rnd); + const double uz = u * zetan; + + if (uz < 1.0) + return 0; + if (uz < 1.0 + std::pow(0.5, theta)) + return 1; + + // 1-origin を想定した式なので 0-origin に揃える + const double val = std::floor( + static_cast(n) * std::pow(eta * u - eta + 1.0, alpha)); + // 範囲ガード + if (val < 0.0) + return 0; + if (val >= static_cast(n)) + return n - 1; + return static_cast(val); +} + +YCSB::YCSB(double read_ratio, double skew, uint64 db_size, uint64 hot_records) + : read_ratio_(read_ratio), + skew_(skew), + db_size_(db_size), + hot_records_(hot_records), + rnd_() {} + +void YCSB::InitializeStorage(Storage *storage, Configuration *config) const +{ + for (uint64 i = 0; i < db_size_; i++) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", i); + if (config->LookupPartition(key_str) == config->this_node_id) + { + Value *value = new Value(RandomString(128), 0); + storage->PutObject(key_str, value); + } + } + // 事前にキー候補を前計算(遅延初期化でも可) + const_cast(this)->PrecomputeKeys(config); +} + +// 事前計算:各パーティションに属する「非ホット」キーの番号リストを作成 +void YCSB::PrecomputeKeys(Configuration *config) +{ + keys_per_partition_.clear(); + keys_per_partition_.resize(config->all_nodes.size()); + for (uint64_t i = hot_records_; i < db_size_; ++i) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", i); + int partition_id = config->LookupPartition(key_str); + keys_per_partition_[partition_id].push_back(i); + } + keys_ready_.store(true, std::memory_order_release); + std::cout << "Key precomputation complete." << std::endl; +} + +TxnProto *YCSB::NewTxn(int64 txn_id, int /*txn_type*/, std::string /*args*/, + Configuration *config) const +{ + // 単一ノードなら常にSP。マルチノード時は 1% を MP に(従来仕様) + const double mp_ratio = 0.01; + if (config->all_nodes.size() == 1 || U01(rnd_) >= mp_ratio) + { + uint32 part = rnd_.Uniform(static_cast(config->all_nodes.size())); + return YCSBTxnSP(txn_id, part, config); + } + else + { + uint32 part1 = rnd_.Uniform(static_cast(config->all_nodes.size())); + uint32 part2; + do + { + part2 = rnd_.Uniform(static_cast(config->all_nodes.size())); + } while (part1 == part2); + return YCSBTxnMP(txn_id, part1, part2, config); + } +} + +int YCSB::Execute(TxnProto * /*txn*/, StorageManager * /*storage*/) const +{ + // Calvin の YCSB は実行ロジックが StorageManager 経由で処理されるためここは空 + return 0; +} + +// ランダムキー抽選(前計算リストから重複ナシで取り出し) +void YCSB::GetRandomKeys(std::set &keys, int num_keys, uint32 part) const +{ + keys.clear(); + if (!keys_ready_.load(std::memory_order_acquire)) + return; + const auto &key_candidates = keys_per_partition_[part]; + if (key_candidates.empty()) + return; + + while ((int)keys.size() < num_keys) + { + uint64_t index = rnd_.Uniform(static_cast(key_candidates.size())); + keys.insert(key_candidates[index]); + } +} + +TxnProto *YCSB::YCSBTxnSP(int64 txn_id, uint32 part, Configuration *config) const +{ + if (!keys_ready_.load(std::memory_order_acquire)) + const_cast(this)->PrecomputeKeys(config); + + TxnProto *txn = new TxnProto(); + txn->set_txn_id(txn_id); + txn->set_txn_type(YCSB_TXN_SP); + + const bool is_read = (U01(rnd_) < read_ratio_); + + // --- ホットキー 1本(Zipf or 一様)--- + uint64 hot_idx = 0; + if (skew_ > 0.0) + { + hot_idx = ZipfSample(rnd_, static_cast(hot_records_), skew_); + } + else + { + hot_idx = rnd_.Uniform(static_cast(hot_records_)); + } + char hotkey_str[32]; + snprintf(hotkey_str, sizeof(hotkey_str), "k%lu", hot_idx); + if (is_read) + { + txn->add_read_set(hotkey_str); + } + else + { + txn->add_read_write_set(hotkey_str); + } + + // --- コールド側(パーティションに属する候補から一様抽選、重複なし)--- + std::set keys; + GetRandomKeys(keys, RW_SET_SIZE - 1, part); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + return txn; +} + +TxnProto *YCSB::YCSBTxnMP(int64 txn_id, uint32 part1, uint32 part2, + Configuration *config) const +{ + if (!keys_ready_.load(std::memory_order_acquire)) + const_cast(this)->PrecomputeKeys(config); + + TxnProto *txn = new TxnProto(); + txn->set_txn_id(txn_id); + txn->set_txn_type(YCSB_TXN_MP); + txn->set_multipartition(true); + + const bool is_read = (U01(rnd_) < read_ratio_); + + // --- 各パーティションのホットキー(Zipf or 一様)--- + uint64 hot1 = (skew_ > 0.0) + ? ZipfSample(rnd_, static_cast(hot_records_), skew_) + : rnd_.Uniform(static_cast(hot_records_)); + uint64 hot2 = (skew_ > 0.0) + ? ZipfSample(rnd_, static_cast(hot_records_), skew_) + : rnd_.Uniform(static_cast(hot_records_)); + + char hotkey1_str[32], hotkey2_str[32]; + snprintf(hotkey1_str, sizeof(hotkey1_str), "k%lu", hot1); + snprintf(hotkey2_str, sizeof(hotkey2_str), "k%lu", hot2); + if (is_read) + { + txn->add_read_set(hotkey1_str); + txn->add_read_set(hotkey2_str); + } + else + { + txn->add_read_write_set(hotkey1_str); + txn->add_read_write_set(hotkey2_str); + } + + // --- コールド側:各パーティションから一様抽選 --- + std::set keys; + GetRandomKeys(keys, RW_SET_SIZE / 2 - 1, part1); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + + keys.clear(); + GetRandomKeys(keys, RW_SET_SIZE / 2 - 1, part2); + for (uint64 key : keys) + { + char key_str[32]; + snprintf(key_str, sizeof(key_str), "k%lu", key); + if (is_read) + { + txn->add_read_set(key_str); + } + else + { + txn->add_read_write_set(key_str); + } + } + + return txn; +} + +TxnProto *YCSB::InitializeTxn() const +{ + TxnProto *txn = new TxnProto(); + txn->set_txn_type(YCSB_TXN_INITIALIZE); + return txn; +} diff --git a/src_calvin_ext/applications/ycsb.h b/src_calvin_ext/applications/ycsb.h new file mode 100644 index 00000000..d1f94ae1 --- /dev/null +++ b/src_calvin_ext/applications/ycsb.h @@ -0,0 +1,47 @@ +#ifndef _DB_APPLICATIONS_YCSB_H_ +#define _DB_APPLICATIONS_YCSB_H_ + +#include "applications/application.h" +#include "common/random.hh" // ← Random の定義 +#include +#include +#include +#include + +class YCSB : public Application +{ +public: + YCSB(double read_ratio, double skew, uint64 db_size, uint64 hot_records); + + void InitializeStorage(Storage *storage, Configuration *config) const override; + TxnProto *NewTxn(int64 txn_id, int txn_type, std::string args, Configuration *config) const override; + int Execute(TxnProto *txn, StorageManager *storage) const override; + + // Application の宣言と一致しない環境があるため override は付けない + TxnProto *InitializeTxn() const; + + // 初期化時または遅延でキー候補を前計算 + void PrecomputeKeys(Configuration *config); + +private: + TxnProto *YCSBTxnSP(int64 txn_id, uint32 part, Configuration *config) const; + TxnProto *YCSBTxnMP(int64 txn_id, uint32 part1, uint32 part2, Configuration *config) const; + + // part に属するランダムキーを重複なしで num_keys 個選ぶ + void GetRandomKeys(std::set &keys, int num_keys, uint32 part) const; + + // パラメータ + double read_ratio_; + double skew_; // ← 追加:Zipf 用スキュー (0 なら一様) + uint64 db_size_; + uint64 hot_records_; + + // 乱数生成器(common/random.hh) + mutable Random rnd_; + + // 前計算キャッシュ + mutable std::atomic keys_ready_{false}; + std::vector> keys_per_partition_; +}; + +#endif // _DB_APPLICATIONS_YCSB_H_ diff --git a/src_calvin_3_partitions/backend/Makefile.inc b/src_calvin_ext/backend/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/backend/Makefile.inc rename to src_calvin_ext/backend/Makefile.inc diff --git a/src_calvin_vector_vll/backend/collapsed_versioned_storage.cc b/src_calvin_ext/backend/collapsed_versioned_storage.cc similarity index 69% rename from src_calvin_vector_vll/backend/collapsed_versioned_storage.cc rename to src_calvin_ext/backend/collapsed_versioned_storage.cc index aa0f99f9..6a7c8ca6 100644 --- a/src_calvin_vector_vll/backend/collapsed_versioned_storage.cc +++ b/src_calvin_ext/backend/collapsed_versioned_storage.cc @@ -14,13 +14,13 @@ using std::string; #ifdef TPCCHACK #define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; +// For inserted objects we need to make a representation that is thread-safe +// (i.e. an array). This is kind of hacky, but since this only corresponds +// to TPCC, we'll be okay +Value* NewOrderStore[MAXARRAYSIZE]; +Value* OrderStore[MAXARRAYSIZE]; +Value* OrderLineStore[MAXARRAYSIZE * 15]; +Value* HistoryStore[MAXARRAYSIZE]; #endif Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { @@ -36,14 +36,14 @@ Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { } else { #endif - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; + // Check to see if a match even exists + if (objects_.count(key) != 0) { + for (DataNode* list = objects_[key]; list; list = list->next) { + if (list->txn_id <= txn_id) { + return list->value; + } } } - } #ifdef TPCCHACK } @@ -53,12 +53,12 @@ Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { return NULL; } -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, +bool CollapsedVersionedStorage::PutObject(const Key& key, + Value* value, int64 txn_id) { #ifdef TPCCHACK if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; + OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = value; } else if (key.find("no") != string::npos) { NewOrderStore[txn_id] = value; } else if (key.find("o") != string::npos) { @@ -68,26 +68,26 @@ bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, } else { #endif - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; + // Create the new version to insert into the list + DataNode* item = new DataNode(); + item->txn_id = txn_id; + item->value = value; + item->next = NULL; + + // Is the most recent value a candidate for pruning? + DataNode* current; + if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { + int64 most_recent = current->txn_id; + + if ((most_recent > stable_ && txn_id > stable_) || + (most_recent <= stable_ && txn_id <= stable_)) { + item->next = current->next; + delete current; + } else { + item->next = current; + } } - } - objects_[key] = item; + objects_[key] = item; #ifdef TPCCHACK } @@ -117,12 +117,12 @@ bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { objects_[key]->txn_id = txn_id; objects_[key]->value = NULL; - // Otherwise we need to free the head + // Otherwise we need to free the head } else if (list != NULL && objects_[key] == list) { delete objects_[key]; objects_[key] = NULL; - // Lastly, we may only want to free the tail + // Lastly, we may only want to free the tail } else if (list != NULL) { delete list; objects_[key]->next = NULL; @@ -133,8 +133,8 @@ bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { int CollapsedVersionedStorage::Checkpoint() { pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); + int thread_status = + pthread_create(&checkpointing_daemon, NULL, &RunCheckpointer, this); return thread_status; } @@ -159,17 +159,13 @@ void CollapsedVersionedStorage::CaptureCheckpoint() { // Write to disk int key_length = key.length(); int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); + fprintf( + checkpoint, "%c%c%c%c%s%c%c%c%c%s", static_cast(key_length >> 24), + static_cast(key_length >> 16), static_cast(key_length >> 8), + static_cast(key_length), key.c_str(), + static_cast(val_length >> 24), + static_cast(val_length >> 16), static_cast(val_length >> 8), + static_cast(val_length), result->c_str()); // Remove object from tree if there's an old version if (it->second->next != NULL) diff --git a/src_calvin_3_partitions/backend/collapsed_versioned_storage.h b/src_calvin_ext/backend/collapsed_versioned_storage.h old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/backend/collapsed_versioned_storage.h rename to src_calvin_ext/backend/collapsed_versioned_storage.h diff --git a/src_calvin_4_partitions/backend/fetching_storage.cc b/src_calvin_ext/backend/fetching_storage.cc old mode 100755 new mode 100644 similarity index 85% rename from src_calvin_4_partitions/backend/fetching_storage.cc rename to src_calvin_ext/backend/fetching_storage.cc index f5514372..e4cedc3e --- a/src_calvin_4_partitions/backend/fetching_storage.cc +++ b/src_calvin_ext/backend/fetching_storage.cc @@ -4,6 +4,7 @@ // main memory, disk, and swapping algorithms. #include "backend/fetching_storage.h" +#include "common/definitions.hh" typedef FetchingStorage::Latch Latch; @@ -26,8 +27,7 @@ FetchingStorage::FetchingStorage() { // 1 MILLION LATCHES! latches_ = new Latch[1000000]; - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); + pthread_create(&gc_thread_, NULL, RunGCThread, reinterpret_cast(this)); } FetchingStorage::~FetchingStorage() { @@ -45,22 +45,22 @@ Latch* FetchingStorage::LatchFor(const Key& key) { } void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); + char path[255]; + snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); + char key_c_str[255]; + memset(&key_c_str, 0, 255); + readlink(path, key_c_str, 255); + *key = string((strrchr(key_c_str, '/') + 1)); } -void* FetchingStorage::RunGCThread(void *arg) { +void* FetchingStorage::RunGCThread(void* arg) { FetchingStorage* storage = reinterpret_cast(arg); while (true) { double start_time = GetTime(); for (int i = COLD_CUTOFF; i < 1000000; i++) { storage->HardUnfetch(IntToString(i)); } - usleep(static_cast(1000000*(GetTime()-start_time))); + usleep(static_cast(1000000 * (GetTime() - start_time))); } return NULL; } @@ -77,7 +77,8 @@ Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { assert(latch->active_requests > 0); pthread_mutex_unlock(&latch->lock_); // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} + while (latch->state == FETCHING) { + } return main_memory_->ReadObject(key); } @@ -157,7 +158,7 @@ bool FetchingStorage::HardUnfetch(const Key& key) { if (active_requests == 0 && previous_state == IN_MEMORY) { Value* result = main_memory_->ReadObject(key); int len = strlen(result->c_str()); - char* c_result = new char[len+1]; + char* c_result = new char[len + 1]; strcpy(c_result, result->c_str()); return FilePut(key, c_result, len); } else { @@ -182,8 +183,8 @@ bool FetchingStorage::Unfetch(const Key& key) { ///////////////// Asynchronous Callbacks //////////////////////// void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; + struct aiocb* req; + req = (struct aiocb*)sigval.sival_ptr; /* Did the request complete? */ if (aio_error(req) == 0) { /* Request completed successfully, get the return status */ @@ -209,8 +210,8 @@ void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { } void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; + struct aiocb* req; + req = (struct aiocb*)sigval.sival_ptr; /* Did the request complete? */ if (aio_error(req) == 0) { /* Request completed successfully, get the return status */ @@ -225,8 +226,8 @@ void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { State state = latch->state; pthread_mutex_unlock(&latch->lock_); if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; + store->main_memory_->DeleteObject(key); + latch->state = ON_DISK; } close(req->aio_fildes); delete[] reinterpret_cast(req->aio_buf); @@ -242,7 +243,7 @@ void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { bool FetchingStorage::FileRead(const Key& key, char* result, int size) { string fileName(STORAGE_PATH); fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); + int fd = open(fileName.c_str(), O_RDONLY | O_NONBLOCK); if (fd == -1) return false; return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; @@ -252,14 +253,17 @@ bool FetchingStorage::FilePut(const Key& key, char* value, int size) { string fileName(STORAGE_PATH); fileName.append(key); mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); + int fd = + open(fileName.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_NONBLOCK, mode); if (fd == -1) return false; return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; } -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { +aiocb* FetchingStorage::generateControlBlock(int fd, + char* buf, + const int size, + Operation op) { aiocb* aiocbp = new aiocb(); aiocbp->aio_fildes = fd; aiocbp->aio_offset = 0; @@ -269,11 +273,9 @@ aiocb* FetchingStorage::generateControlBlock( /* Link the AIO request with a thread callback */ aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; + aiocbp->aio_sigevent.sigev_notify_function = &PrefetchCompletionHandler; else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; + aiocbp->aio_sigevent.sigev_notify_function = &UnfetchCompletionHandler; aiocbp->aio_sigevent.sigev_notify_attributes = NULL; aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; return aiocbp; diff --git a/src_calvin_4_partitions/backend/fetching_storage.h b/src_calvin_ext/backend/fetching_storage.h old mode 100755 new mode 100644 similarity index 87% rename from src_calvin_4_partitions/backend/fetching_storage.h rename to src_calvin_ext/backend/fetching_storage.h index 273dc04c..c63541a0 --- a/src_calvin_4_partitions/backend/fetching_storage.h +++ b/src_calvin_ext/backend/fetching_storage.h @@ -15,6 +15,8 @@ #include #include +#include + #include "common/utils.h" #include "backend/storage.h" #include "backend/simple_storage.h" @@ -22,8 +24,6 @@ #define PAGE_SIZE 1000 #define STORAGE_PATH "../db/storage/" -#define COLD_CUTOFF 990000 - class FetchingStorage : public Storage { public: static FetchingStorage* BuildStorage(); @@ -31,15 +31,13 @@ class FetchingStorage : public Storage { virtual Value* ReadObject(const Key& key, int64 txn_id = 0); virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); + virtual bool Prefetch(const Key& key, double* wait_time); + virtual bool Unfetch(const Key& key); bool HardUnfetch(const Key& key); // Latch object that stores a counter for readlocks and a boolean for write // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; + enum State { UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING }; class Latch { public: int active_requests; // Can be as many as you want. @@ -52,7 +50,7 @@ class FetchingStorage : public Storage { pthread_mutex_init(&lock_, NULL); } }; - Latch* LatchFor(const Key &key); + Latch* LatchFor(const Key& key); static void PrefetchCompletionHandler(sigval_t sigval); static void UnfetchCompletionHandler(sigval_t sigval); static void GetKey(int fd, Key* key); @@ -65,9 +63,7 @@ class FetchingStorage : public Storage { * is being placed here. After November 1, we can swap out backends. */ - enum Operation { - FETCH, RELEASE - }; + enum Operation { FETCH, RELEASE }; // Registers an asynchronous read. bool FileRead(const Key& key, char* result, int size); @@ -87,7 +83,7 @@ class FetchingStorage : public Storage { Latch* latches_; // GC thread stuff. - static void* RunGCThread(void *arg); + static void* RunGCThread(void* arg); pthread_t gc_thread_; }; #endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_traditional_2pl_2pc/backend/simple_storage.cc b/src_calvin_ext/backend/simple_storage.cc old mode 100755 new mode 100644 similarity index 94% rename from src_traditional_2pl_2pc/backend/simple_storage.cc rename to src_calvin_ext/backend/simple_storage.cc index 4d7760d4..73196601 --- a/src_traditional_2pl_2pc/backend/simple_storage.cc +++ b/src_calvin_ext/backend/simple_storage.cc @@ -1,5 +1,5 @@ // Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) +// Author: Kun Ren (kun.ren@yale.edu) // // A simple implementation of the storage interface using an stl map. diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.h b/src_calvin_ext/backend/simple_storage.h similarity index 85% rename from src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.h rename to src_calvin_ext/backend/simple_storage.h index f15f69e9..00fffe67 100644 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.h +++ b/src_calvin_ext/backend/simple_storage.h @@ -19,8 +19,8 @@ class SimpleStorage : public Storage { virtual ~SimpleStorage() {} // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } + virtual bool Prefetch(const Key& key, double* wait_time) { return false; } + virtual bool Unfetch(const Key& key) { return false; } virtual Value* ReadObject(const Key& key, int64 txn_id = 0); virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); virtual bool DeleteObject(const Key& key, int64 txn_id = 0); @@ -32,7 +32,5 @@ class SimpleStorage : public Storage { private: unordered_map objects_; pthread_mutex_t mutex_; - }; #endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_dependent_remote_index/backend/storage.h b/src_calvin_ext/backend/storage.h similarity index 93% rename from src_dependent_remote_index/backend/storage.h rename to src_calvin_ext/backend/storage.h index fa2e4efb..cbbc48bf 100644 --- a/src_dependent_remote_index/backend/storage.h +++ b/src_calvin_ext/backend/storage.h @@ -19,11 +19,11 @@ class Storage { // Loads object specified by 'key' into memory if currently stored // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; + virtual bool Prefetch(const Key& key, double* wait_time) = 0; // Unfetch object on memory, writing it off to disk, asynchronously or // otherwise. - virtual bool Unfetch(const Key &key) = 0; + virtual bool Unfetch(const Key& key) = 0; // If the object specified by 'key' exists, copies the object into '*result' // and returns true. If the object does not exist, false is returned. @@ -46,4 +46,3 @@ class Storage { }; #endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_dependent_remote_index/backend/storage_manager.cc b/src_calvin_ext/backend/storage_manager.cc similarity index 91% rename from src_dependent_remote_index/backend/storage_manager.cc rename to src_calvin_ext/backend/storage_manager.cc index 7de4ff40..154c4cfc 100644 --- a/src_dependent_remote_index/backend/storage_manager.cc +++ b/src_calvin_ext/backend/storage_manager.cc @@ -12,10 +12,14 @@ #include "proto/txn.pb.h" #include "proto/message.pb.h" -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { +StorageManager::StorageManager(Configuration* config, + Connection* connection, + Storage* actual_storage, + TxnProto* txn) + : configuration_(config), + connection_(connection), + actual_storage_(actual_storage), + txn_(txn) { MessageProto message; // If reads are performed at this node, execute local reads and broadcast @@ -111,4 +115,3 @@ bool StorageManager::DeleteObject(const Key& key) { else return true; // Not this node's problem. } - diff --git a/src_dependent_remote_index/backend/storage_manager.h b/src_calvin_ext/backend/storage_manager.h similarity index 93% rename from src_dependent_remote_index/backend/storage_manager.h rename to src_calvin_ext/backend/storage_manager.h index a374017a..67424795 100644 --- a/src_dependent_remote_index/backend/storage_manager.h +++ b/src_calvin_ext/backend/storage_manager.h @@ -40,8 +40,10 @@ class TxnProto; class StorageManager { public: // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); + StorageManager(Configuration* config, + Connection* connection, + Storage* actual_storage, + TxnProto* txn); ~StorageManager(); @@ -58,7 +60,7 @@ class StorageManager { // this node. bool writer; -// private: + // private: friend class DeterministicScheduler; // Pointer to the configuration object for this node. @@ -80,8 +82,6 @@ class StorageManager { unordered_map objects_; vector remote_reads_; - }; #endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_calvin_4_partitions/backend/versioned_storage.h b/src_calvin_ext/backend/versioned_storage.h old mode 100755 new mode 100644 similarity index 90% rename from src_calvin_4_partitions/backend/versioned_storage.h rename to src_calvin_ext/backend/versioned_storage.h index 24c6223b..45c4ee35 --- a/src_calvin_4_partitions/backend/versioned_storage.h +++ b/src_calvin_ext/backend/versioned_storage.h @@ -37,10 +37,9 @@ class VersionedStorage : public Storage { // specified (i.e. txn_id). virtual int Checkpoint() = 0; - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } + virtual bool Prefetch(const Key& key, double* wait_time) { return false; } + virtual bool Unfetch(const Key& key) { return false; } }; #endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_calvin_3_partitions/common/Makefile.inc b/src_calvin_ext/common/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/common/Makefile.inc rename to src_calvin_ext/common/Makefile.inc diff --git a/src_traditional_2pl_2pc_4_partitions/common/configuration.cc b/src_calvin_ext/common/configuration.cc old mode 100755 new mode 100644 similarity index 81% rename from src_traditional_2pl_2pc_4_partitions/common/configuration.cc rename to src_calvin_ext/common/configuration.cc index aede7dab..af1722f8 --- a/src_traditional_2pl_2pc_4_partitions/common/configuration.cc +++ b/src_calvin_ext/common/configuration.cc @@ -34,17 +34,12 @@ int Configuration::LookupPartition(const Key& key) const { bool Configuration::WriteToFile(const string& filename) const { FILE* fp = fopen(filename.c_str(), "w"); if (fp == NULL) - return false; + return false; for (map::const_iterator it = all_nodes.begin(); it != all_nodes.end(); ++it) { Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - node->node_id, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); + fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", it->first, node->replica_id, + node->partition_id, node->cores, node->host.c_str(), node->port); } fclose(fp); return true; @@ -88,11 +83,11 @@ void Configuration::ProcessConfigLine(char key[], char value[]) { // Parse additional node addributes. char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); + node->replica_id = atoi(strtok_r(value, ":", &tok)); node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); + node->cores = atoi(strtok_r(NULL, ":", &tok)); + const char* host = strtok_r(NULL, ":", &tok); + node->port = atoi(strtok_r(NULL, ":", &tok)); // Translate hostnames to IP addresses. string ip; @@ -105,16 +100,13 @@ void Configuration::ProcessConfigLine(char key[], char value[]) { char buf[32]; memmove(&n, ent->h_addr_list[0], ent->h_length); n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); + snprintf(buf, sizeof(buf), "%u.%u.%u.%u", n >> 24, (n >> 16) & 0xff, + (n >> 8) & 0xff, n & 0xff); ip = buf; } } node->host = ip; all_nodes[node->node_id] = node; - } } - diff --git a/src_calvin_4_partitions/common/configuration.h b/src_calvin_ext/common/configuration.h old mode 100755 new mode 100644 similarity index 98% rename from src_calvin_4_partitions/common/configuration.h rename to src_calvin_ext/common/configuration.h index d7ce7fb5..9712604b --- a/src_calvin_4_partitions/common/configuration.h +++ b/src_calvin_ext/common/configuration.h @@ -1,5 +1,6 @@ // Author: Shu-chun Weng (scweng@cs.yale.edu) // Author: Alexander Thomson (thomson@cs.yale.edu) +// Author: Kun Ren (kun@cs.yale.edu) // // Each node in the system has a Configuration, which stores the identity of // that node, the system's current execution mode, and the set of all currently @@ -27,10 +28,8 @@ #include #include - #include "common/types.h" - using std::map; using std::string; using std::vector; @@ -92,4 +91,3 @@ class Configuration { }; #endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_calvin_3_partitions/common/configuration_test.conf b/src_calvin_ext/common/configuration_test.conf old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/common/configuration_test.conf rename to src_calvin_ext/common/configuration_test.conf diff --git a/src_calvin_3_partitions/common/configuration_test_one_node.conf b/src_calvin_ext/common/configuration_test_one_node.conf old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/common/configuration_test_one_node.conf rename to src_calvin_ext/common/configuration_test_one_node.conf diff --git a/src_calvin_vector_vll/common/connection.cc b/src_calvin_ext/common/connection.cc old mode 100755 new mode 100644 similarity index 76% rename from src_calvin_vector_vll/common/connection.cc rename to src_calvin_ext/common/connection.cc index d6a1e7d5..98073793 --- a/src_calvin_vector_vll/common/connection.cc +++ b/src_calvin_ext/common/connection.cc @@ -8,12 +8,17 @@ #include "common/configuration.h" #include "common/utils.h" +#include "common/definitions.hh" +#include "common/debug.hh" using zmq::socket_t; ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { + : configuration_(config), + context_(1), + new_connection_channel_(NULL), + delete_connection_channel_(NULL), + deconstructor_invoked_(false) { // Lookup port. (Pick semi-arbitrary port if node id < 0). if (config->this_node_id < 0) port_ = config->all_nodes.begin()->second->port; @@ -33,7 +38,7 @@ ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) // Wait for other nodes to bind sockets before connecting to them. Spin(0.1); -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; + send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; // Connect to remote outgoing sockets. for (map::const_iterator it = config->all_nodes.begin(); @@ -47,22 +52,19 @@ send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; } } -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + cpu_set_t cpuset; + pthread_attr_t attr; + pthread_attr_init(&attr); + // pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); + CPU_ZERO(&cpuset); + CPU_SET(MULTIPLEXER_CORE, &cpuset); + pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); + pthread_create(&thread_, &attr, RunMultiplexer, + reinterpret_cast(this)); // Initialize mutex for future calls to NewConnection. pthread_mutex_init(&new_connection_mutex_, NULL); @@ -92,13 +94,15 @@ ConnectionMultiplexer::~ConnectionMultiplexer() { it != inproc_out_.end(); ++it) { delete it->second; } - - for (unordered_map*>::iterator it = remote_result_.begin(); + + for (unordered_map*>::iterator it = + remote_result_.begin(); it != remote_result_.end(); ++it) { delete it->second; } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); + + for (unordered_map*>::iterator it = + link_unlink_queue_.begin(); it != link_unlink_queue_.end(); ++it) { delete it->second; } @@ -113,7 +117,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel) { // Wait for the Run() loop to create the Connection object. (It will reset // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} + while (new_connection_channel_ != NULL) { + } Connection* connection = new_connection_; new_connection_ = NULL; @@ -124,7 +129,9 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel) { return connection; } -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { +Connection* ConnectionMultiplexer::NewConnection( + const string& channel, + AtomicQueue** aa) { // Disallow concurrent calls to NewConnection/~Connection. pthread_mutex_lock(&new_connection_mutex_); remote_result_[channel] = *aa; @@ -133,7 +140,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQu // Wait for the Run() loop to create the Connection object. (It will reset // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} + while (new_connection_channel_ != NULL) { + } Connection* connection = new_connection_; new_connection_ = NULL; @@ -144,6 +152,8 @@ Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQu } void ConnectionMultiplexer::Run() { + PrintCpu("Multiplexer", 0); + MessageProto message; zmq::message_t msg; @@ -154,7 +164,8 @@ void ConnectionMultiplexer::Run() { // Channel name already in use. Report an error and set new_connection_ // (which NewConnection() will return) to NULL. std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; + << (*new_connection_channel_) << "\n" + << std::flush; new_connection_ = NULL; } else { // Channel name is not already in use. Create a new Connection object @@ -171,10 +182,10 @@ void ConnectionMultiplexer::Run() { new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); new_connection_->socket_in_->connect(endpoint); new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); + new_connection_->socket_out_->connect( + "inproc://__inproc_in_endpoint__"); - // Forward on any messages sent to this channel before it existed. + // Forward on any messages sent to this channel before it existed. vector::iterator i; for (i = undelivered_messages_[*new_connection_channel_].begin(); i != undelivered_messages_[*new_connection_channel_].end(); ++i) { @@ -183,13 +194,13 @@ void ConnectionMultiplexer::Run() { undelivered_messages_.erase(*new_connection_channel_); } - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); + if ((new_connection_channel_->substr(0, 9) == "scheduler") && + (new_connection_channel_->substr(9, 1) != "_")) { + link_unlink_queue_[*new_connection_channel_] = + new AtomicQueue(); } // Reset request variable. new_connection_channel_ = NULL; - } // Serve any pending (valid) connection deletion request. @@ -213,43 +224,41 @@ void ConnectionMultiplexer::Run() { // local Link/UnlinkChannel requests. if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); + // Normal message. Forward appropriately. + Send(message); } - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - + for (unordered_map*>::iterator it = + link_unlink_queue_.begin(); + it != link_unlink_queue_.end(); ++it) { + MessageProto message; + bool got_it = it->second->Pop(&message); + if (got_it == true) { + if (message.type() == MessageProto::LINK_CHANNEL) { + remote_result_[message.channel_request()] = remote_result_[it->first]; + // Forward on any messages sent to this channel before it existed. + vector::iterator i; + for (i = undelivered_messages_[message.channel_request()].begin(); + i != undelivered_messages_[message.channel_request()].end(); + ++i) { + Send(*i); + } + undelivered_messages_.erase(message.channel_request()); + } else if (message.type() == MessageProto::UNLINK_CHANNEL) { + remote_result_.erase(message.channel_request()); + } + } + } } } // Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { +void* ConnectionMultiplexer::RunMultiplexer(void* multiplexer) { reinterpret_cast(multiplexer)->Run(); return NULL; } void ConnectionMultiplexer::Send(const MessageProto& message) { - if (message.type() == MessageProto::READ_RESULT) { if (remote_result_.count(message.destination_channel()) > 0) { remote_result_[message.destination_channel()]->Push(message); @@ -257,20 +266,18 @@ void ConnectionMultiplexer::Send(const MessageProto& message) { undelivered_messages_[message.destination_channel()].push_back(message); } } else { - // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); // Send message. if (message.destination_node() == configuration_->this_node_id) { // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. + // message on, else store it to be delivered if the channel is ever + // created. if (inproc_out_.count(message.destination_channel()) > 0) inproc_out_[message.destination_channel()]->send(msg); else @@ -278,10 +285,10 @@ void ConnectionMultiplexer::Send(const MessageProto& message) { } else { // Message is addressed to valid remote node. Channel validity will be // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); + pthread_mutex_lock(&send_mutex_[message.destination_node()]); remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } + pthread_mutex_unlock(&send_mutex_[message.destination_node()]); + } } } @@ -304,7 +311,8 @@ Connection::~Connection() { // Wait for the Run() loop to delete its socket for this Connection object. // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} + while (multiplexer_->delete_connection_channel_ != NULL) { + } // Allow future calls to NewConnection/~Connection. pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); @@ -314,11 +322,9 @@ void Connection::Send(const MessageProto& message) { // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); // Send message. socket_out_->send(msg); } @@ -327,12 +333,10 @@ void Connection::Send1(const MessageProto& message) { // Prepare message. string* message_string = new string(); message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); + zmq::message_t msg( + reinterpret_cast(const_cast(message_string->data())), + message_string->size(), DeleteString, message_string); + pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); multiplexer()->remote_out_[message.destination_node()]->send(msg); pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); } @@ -376,4 +380,3 @@ void Connection::UnlinkChannel(const string& channel) { m.set_channel_request(channel); multiplexer()->link_unlink_queue_[channel_]->Push(m); } - diff --git a/src_calvin_4_partitions/common/connection.h b/src_calvin_ext/common/connection.h old mode 100755 new mode 100644 similarity index 97% rename from src_calvin_4_partitions/common/connection.h rename to src_calvin_ext/common/connection.h index 99fa7c0e..47072b28 --- a/src_calvin_4_partitions/common/connection.h +++ b/src_calvin_ext/common/connection.h @@ -1,6 +1,6 @@ // Author: Kun Ren (kun.ren@yale.edu) -// Author: Shu-chun Weng (scweng@cs.yale.edu) // Author: Alexander Thomson (thomson@cs.yale.edu) +// Author: Shu-chun Weng (scweng@cs.yale.edu) // // Library for handling messaging between system nodes. Each node generally owns // a ConnectionMultiplexer object as well as a Configuration object. @@ -16,7 +16,6 @@ #include #include - #include "common/zmq.hpp" #include "proto/message.pb.h" #include "common/utils.h" @@ -47,8 +46,9 @@ class ConnectionMultiplexer { // the channel name is already in use, in which case NULL is returned. The // caller (not the multiplexer) owns of the newly created Connection object. Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); + + Connection* NewConnection(const string& channel, + AtomicQueue** aa); zmq::context_t* context() { return &context_; } @@ -60,7 +60,7 @@ class ConnectionMultiplexer { void Run(); // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); + static void* RunMultiplexer(void* multiplexer); // TODO(alex): Comments. void Send(const MessageProto& message); @@ -92,9 +92,9 @@ class ConnectionMultiplexer { // Sockets for forwarding messages to Connections. Keyed by channel // name. Type = ZMQ_PUSH. unordered_map inproc_out_; - + unordered_map*> remote_result_; - + unordered_map*> link_unlink_queue_; // Stores messages addressed to local channels that do not exist at the time @@ -106,9 +106,9 @@ class ConnectionMultiplexer { // Protects concurrent calls to NewConnection(). pthread_mutex_t new_connection_mutex_; - + pthread_mutex_t* send_mutex_; - + // Specifies a requested channel. Null if there is no outstanding new // connection request. const string* new_connection_channel_; @@ -137,7 +137,7 @@ class Connection { // Sends 'message' to the Connection specified by // 'message.destination_node()' and 'message.destination_channel()'. void Send(const MessageProto& message); - + void Send1(const MessageProto& message); // Loads the next incoming MessageProto into 'message'. Returns true, unless @@ -194,4 +194,3 @@ class Connection { }; #endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_calvin_ext/common/debug.hh b/src_calvin_ext/common/debug.hh new file mode 100644 index 00000000..c20fe396 --- /dev/null +++ b/src_calvin_ext/common/debug.hh @@ -0,0 +1,9 @@ +#pragma once + +#include +#include +#include // これを追加 + +static void PrintCpu(std::string worker_name, int id) { + std::cout << worker_name << " is on cpu " << sched_getcpu() << std::endl; +} \ No newline at end of file diff --git a/src_calvin_ext/common/definitions.hh b/src_calvin_ext/common/definitions.hh new file mode 100644 index 00000000..22a9272b --- /dev/null +++ b/src_calvin_ext/common/definitions.hh @@ -0,0 +1,11 @@ +#pragma once + +#define MAX_ACTIVE_TXNS 2000 // default 2000 +#define LOCK_BATCH_SIZE 100 // default 100 + +#define HOT 10 + +#define COLD_CUTOFF 990000 // default 990000 + +#define RW_SET_SIZE 1 // MUST BE EVEN, default 10 +#define DB_SIZE 1000000 // default 1000000 \ No newline at end of file diff --git a/src_calvin_ext/common/inline.hh b/src_calvin_ext/common/inline.hh new file mode 100644 index 00000000..936b3b69 --- /dev/null +++ b/src_calvin_ext/common/inline.hh @@ -0,0 +1,3 @@ +#pragma once + +#define INLINE __attribute__((always_inline)) inline diff --git a/src_calvin_ext/common/random.hh b/src_calvin_ext/common/random.hh new file mode 100644 index 00000000..350c10c5 --- /dev/null +++ b/src_calvin_ext/common/random.hh @@ -0,0 +1,132 @@ +/** + * This algorithm is originally developed + * by David Blackman and Sebastiano Vigna (vigna@acm.org) + * http://xoroshiro.di.unimi.it/xoroshiro128plus.c + * + * And Tanabe Takayuki custmized. + * + * Additional methods for specific distributions (Uniform, Zipf) were integrated. + */ + +#pragma once + +#include +#include +#include + +class Xoroshiro128Plus +{ +public: + Xoroshiro128Plus(uint64_t seed = 0) + { + if (seed == 0) + { + std::random_device rd; + s[0] = (static_cast(rd()) << 32) | rd(); + s[1] = (static_cast(rd()) << 32) | rd(); + } + else + { + s[0] = splitMix64(seed); + s[1] = splitMix64(s[0]); + } + } + + uint64_t s[2]; + + // Generates the next 64-bit pseudo-random number. + uint64_t next(void) + { + const uint64_t s0 = s[0]; + uint64_t s1 = s[1]; + const uint64_t result = s0 + s1; + + s1 ^= s0; + s[0] = rotl(s0, 24) ^ s1 ^ (s1 << 16); // a, b + s[1] = rotl(s1, 37); // c + + return result; + } + + // Overload operator() to be used as a standard generator. + uint64_t operator()() { return next(); } + + // ------------------- ここから追加するメソッド ------------------- + + // Generates a random double in [0.0, 1.0). + double NextUniform() + { + return (next() >> 11) * (1.0 / (1LL << 53)); + } + + // Generates a random integer uniformly distributed in [0, n-1]. + uint64_t Uniform(uint64_t n) + { + return next() % n; + } + + // Generates a random integer from a Zipfian distribution in [0, n-1]. + uint64_t Zipf(uint64_t n, double theta) + { + if (!zipf_generator_initialized || n != zipf_n || theta != zipf_theta) + { + init_zipf_generator(n, theta); + } + + double alpha = 1.0 / (1.0 - theta); + double zetan = zipf_zeta_n; + double eta = (1.0 - std::pow(2.0 / n, 1.0 - theta)) / + (1.0 - zipf_zeta_2 / zetan); + + double u = NextUniform(); + double uz = u * zetan; + + if (uz < 1.0) + return 0; + if (uz < 1.0 + std::pow(0.5, theta)) + return 1; + + return static_cast(n * std::pow(eta * u - eta + 1.0, alpha)); + } + + // ------------------- 追加メソッドここまで ------------------- + +private: + // For seeding the generator. + uint64_t splitMix64(uint64_t seed) + { + uint64_t z = (seed += 0x9e3779b97f4a7c15); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) * 0x94d049bb133111eb; + return z ^ (z >> 31); + } + + inline uint64_t rotl(const uint64_t x, int k) + { + return (x << k) | (x >> (64 - k)); + } + + // --- Zipfian distribution helper variables and functions --- + // mutable allows these to be modified even in const methods. + mutable bool zipf_generator_initialized = false; + mutable uint64_t zipf_n = 0; + mutable double zipf_theta = 0.0; + mutable double zipf_zeta_n = 0.0; + mutable double zipf_zeta_2 = 0.0; + + // Pre-computes constants for the Zipfian distribution. + void init_zipf_generator(uint64_t n, double theta) const + { + Xoroshiro128Plus *non_const_this = const_cast(this); + non_const_this->zipf_zeta_n = 0; + for (uint64_t i = 0; i < n; i++) + { + non_const_this->zipf_zeta_n += 1 / std::pow(i + 1, theta); + } + non_const_this->zipf_zeta_2 = 1 + (1 / std::pow(2, theta)); + non_const_this->zipf_n = n; + non_const_this->zipf_theta = theta; + non_const_this->zipf_generator_initialized = true; + } +}; +using Random = Xoroshiro128Plus; \ No newline at end of file diff --git a/src_calvin_3_partitions/common/test_template.cc b/src_calvin_ext/common/test_template.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_3_partitions/common/test_template.cc rename to src_calvin_ext/common/test_template.cc index 9dc11631..4f4175d3 --- a/src_calvin_3_partitions/common/test_template.cc +++ b/src_calvin_ext/common/test_template.cc @@ -11,4 +11,3 @@ TEST(MyTest) { int main(int argc, char** argv) { MyTest(); } - diff --git a/src_calvin_ext/common/testing.h b/src_calvin_ext/common/testing.h new file mode 100644 index 00000000..d56ba735 --- /dev/null +++ b/src_calvin_ext/common/testing.h @@ -0,0 +1,98 @@ +// Author: Alexander Thomson (thomson@cs.yale.edu) +// +// NOTE(scw): This file is deprecated. The project is migrating to googletest. +// +// Testing framework similar to Google unittests. Example: +// +// TEST(SampleObjectTest) { +// SampleObject a(4,8,15); +// SampleObject b(16,23,42); +// +// EXPECT_FALSE(a.TransformCalled()); +// +// a.Transform(); +// EXPECT_TRUE(a.TransformCalled()); +// +// EXPECT_EQ(b,a); +// +// END; +// } +// +// int main(int argc, char** argv) { +// SampleObjectTest(); +// } +// + +#ifndef _DB_COMMON_TESTING_H_ +#define _DB_COMMON_TESTING_H_ + +#warning Using deprecated common/test.h module, use googletest instead. + +#include +#include + +using namespace std; // Don't do this at home, kids. + +// Global variable tracking whether current test has failed. +bool __failed_; + +#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) +#define CHECK(T, MSG) \ + do { \ + if (!(T)) { \ + __failed_ = true; \ + WARN(MSG); \ + } \ + } while (0) + +#define LINE cout << "[ " << __FUNCTION__ << " ] " + +#define EXPECT_TRUE(T) \ + do { \ + if (!(T)) { \ + __failed_ = true; \ + cout << "EXPECT_TRUE(" << #T << ") failed at " << __FILE__ << ":" \ + << __LINE__ << "\n"; \ + } \ + } while (0) + +#define EXPECT_FALSE(T) \ + do { \ + if (T) { \ + __failed_ = true; \ + cout << "EXPECT_FALSE(" << #T << ") failed at " << __FILE__ << ":" \ + << __LINE__ << "\n"; \ + } \ + } while (0) + +#define EXPECT_EQ(A, B) \ + do { \ + if ((A) != (B)) { \ + __failed_ = true; \ + cout << "EXPECT_EQ(" << #A << ", " << #B \ + << ") \033[1;31mfailed\033[0m at " << __FILE__ << ":" << __LINE__ \ + << "\n" \ + << "Expected:\n" \ + << A << "\n" \ + << "Actual:\n" \ + << B << "\n"; \ + } \ + } while (0) + +#define TEST(TESTNAME) \ + void TESTNAME() { \ + __failed_ = false; \ + LINE << "\033[1;32mBEGIN\033[0m\n"; \ + do + +#define END \ + if (__failed_) { \ + LINE << "\033[1;31mFAIL\033[0m\n"; \ + } else { \ + LINE << "\033[1;32mPASS\033[0m\n"; \ + } \ + } \ + while (0) \ + ; + +#endif // _DB_COMMON_TESTING_H_ diff --git a/src_calvin_ext/common/types.h b/src_calvin_ext/common/types.h new file mode 100644 index 00000000..c6cb7dde --- /dev/null +++ b/src_calvin_ext/common/types.h @@ -0,0 +1,93 @@ +// Author: Alexander Thomson (thomson@cs.yale.edu) +// Author: Kun Ren (kun@cs.yale.edu) +// +// Defininitions of some common types and constants used in the system. + +#ifndef _DB_COMMON_TYPES_H_ +#define _DB_COMMON_TYPES_H_ + +#include + +#include + +using std::string; + +// Abbreviated signed int types. +typedef int8_t int8; +typedef int16_t int16; +typedef int32_t int32; +typedef int64_t int64; + +// Abbreviated unsigned int types. +typedef uint8_t uint8; +typedef uint16_t uint16; +typedef uint32_t uint32; +typedef uint64_t uint64; + +// 'bytes' is an arbitrary sequence of bytes, represented as a string. +typedef string bytes; + +// Convenience functions for converting between (signed) int and 'bytes' types. +static inline bytes PackInt8(int8 x) { + return bytes((const char*)&x, 1); +} +static inline bytes PackInt16(int16 x) { + return bytes((const char*)&x, 2); +} +static inline bytes PackInt32(int32 x) { + return bytes((const char*)&x, 4); +} +static inline bytes PackInt64(int64 x) { + return bytes((const char*)&x, 8); +} +static inline int8 UnpackInt8(bytes s) { + return *((int8*)(s.data())); +} +static inline int16 UnpackInt16(bytes s) { + return *((int16*)(s.data())); +} +static inline int32 UnpackInt32(bytes s) { + return *((int32*)(s.data())); +} +static inline int64 UnpackInt64(bytes s) { + return *((int64*)(s.data())); +} + +// Convenience functions for converting between unsigned int and 'bytes' types. +static inline bytes PackUInt8(uint8 x) { + return bytes((const char*)&x, 1); +} +static inline bytes PackUInt16(uint16 x) { + return bytes((const char*)&x, 2); +} +static inline bytes PackUInt32(uint32 x) { + return bytes((const char*)&x, 4); +} +static inline bytes PackUInt64(uint64 x) { + return bytes((const char*)&x, 8); +} +static inline uint8 UnpackUInt8(bytes s) { + return *((uint8*)(s.data())); +} +static inline uint16 UnpackUInt16(bytes s) { + return *((uint16*)(s.data())); +} +static inline uint32 UnpackUInt32(bytes s) { + return *((uint32*)(s.data())); +} +static inline uint64 UnpackUInt64(bytes s) { + return *((uint64*)(s.data())); +} + +// Key type for database objects. +// Note: if this changes from bytes, the types need to be updated for the +// following fields in .proto files: +// proto/txn.proto: +// TxnProto::'read_set' +// TxnProto::'write_set' +typedef bytes Key; + +// Value type for database objects. +typedef bytes Value; + +#endif // _DB_COMMON_TYPES_H_ diff --git a/src_calvin_3_partitions/common/utils.h b/src_calvin_ext/common/utils.h old mode 100755 new mode 100644 similarity index 89% rename from src_calvin_3_partitions/common/utils.h rename to src_calvin_ext/common/utils.h index ea1021cb..012a83a4 --- a/src_calvin_3_partitions/common/utils.h +++ b/src_calvin_ext/common/utils.h @@ -24,7 +24,11 @@ using std::tr1::unordered_map; #define ASSERTS_ON true -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) +#define DCHECK(ARG) \ + do { \ + if (ASSERTS_ON) \ + assert(ARG); \ + } while (0) // Status code for return values. struct Status { @@ -50,9 +54,12 @@ struct Status { // Pretty printing. string ToString() { string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); + if (code == ERROR) + out.append("Error"); + if (code == OKAY) + out.append("Okay"); + if (code == DONE) + out.append("Done"); if (message.size()) { out.append(": "); out.append(message); @@ -66,19 +73,20 @@ struct Status { static inline double GetTime() { struct timeval tv; gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; + return tv.tv_sec + tv.tv_usec / 1e6; } // Busy-wait for 'duration' seconds. static inline void Spin(double duration) { usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} + // double start = GetTime(); + // while (GetTime() < start + duration) {} } // Busy-wait until GetTime() >= time. static inline void SpinUntil(double time) { - while (GetTime() >= time) {} + while (GetTime() >= time) { + } } // Produces a random alphabet string of the specified length @@ -113,8 +121,8 @@ static inline double StringToDouble(const string& s) { } static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); + double f = (double)rand() / RAND_MAX; + return fMin + f * (fMax - fMin); } // Converts a human-readable numeric sub-string (starting at the 'n'th position @@ -139,9 +147,7 @@ static inline void Noop(void* data, void* hint) {} class Mutex { public: // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } + Mutex() { pthread_mutex_init(&mutex_, NULL); } private: friend class Lock; @@ -158,9 +164,7 @@ class Lock { explicit Lock(Mutex* mutex) : mutex_(mutex) { pthread_mutex_lock(&mutex_->mutex_); } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } + ~Lock() { pthread_mutex_unlock(&mutex_->mutex_); } private: Mutex* mutex_; @@ -175,7 +179,7 @@ class Lock { //////////////////////////////////////////////////////////////// -template +template class AtomicQueue { public: AtomicQueue() { @@ -192,21 +196,19 @@ class AtomicQueue { } // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } + inline bool Empty() { return front_ == back_; } // Atomically pushes 'item' onto the queue. inline void Push(const T& item) { Lock l(&back_mutex_); // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { + if (front_ == (back_ + 1) % size_) { Lock m(&front_mutex_); Lock n(&size_mutex_); uint32 count = (back_ + size_ - front_) % size_; queue_.resize(size_ * 2); for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; + queue_[size_ + i] = queue_[(front_ + i) % size_]; } front_ = size_; back_ = size_ + count; @@ -260,9 +262,7 @@ class AtomicQueue { class MutexRW { public: // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } + MutexRW() { pthread_rwlock_init(&mutex_, NULL); } private: friend class ReadLock; @@ -280,9 +280,7 @@ class ReadLock { explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { pthread_rwlock_rdlock(&mutex_->mutex_); } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } + ~ReadLock() { pthread_rwlock_unlock(&mutex_->mutex_); } private: MutexRW* mutex_; @@ -300,9 +298,7 @@ class WriteLock { explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { pthread_rwlock_wrlock(&mutex_->mutex_); } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } + ~WriteLock() { pthread_rwlock_unlock(&mutex_->mutex_); } private: MutexRW* mutex_; @@ -315,7 +311,7 @@ class WriteLock { WriteLock& operator=(const WriteLock&); }; -template +template class AtomicMap { public: AtomicMap() {} @@ -362,7 +358,7 @@ class AtomicMap { inline void DeleteVAndClear() { WriteLock l(&mutex_); for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { + it != map_.end(); ++it) { delete it->second; } map_.clear(); @@ -378,4 +374,3 @@ class AtomicMap { }; #endif // _DB_COMMON_UTILS_H_ - diff --git a/src_calvin_ext/common/zipf.hh b/src_calvin_ext/common/zipf.hh new file mode 100644 index 00000000..93f24bb5 --- /dev/null +++ b/src_calvin_ext/common/zipf.hh @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "inline.hh" +#include "random.hh" + +using std::cout; +using std::endl; + +// Fast zipf distribution by Jim Gray et al. +class FastZipf { + Xoroshiro128Plus* rnd_; + const size_t nr_; + const double alpha_, zetan_, eta_; + const double threshold_; + + public: + FastZipf(Xoroshiro128Plus* rnd, double theta, size_t nr) + : rnd_(rnd), + nr_(nr), + alpha_(1.0 / (1.0 - theta)), + zetan_(zeta(nr, theta)), + eta_((1.0 - std::pow(2.0 / (double)nr, 1.0 - theta)) / + (1.0 - zeta(2, theta) / zetan_)), + threshold_(1.0 + std::pow(0.5, theta)) { + assert(0.0 <= theta); + assert(theta < 1.0); // 1.0 can not be specified. + } + + // Use this constructor if zeta is pre-calculated. + FastZipf(Xoroshiro128Plus* rnd, double theta, size_t nr, double zetan) + : rnd_(rnd), + nr_(nr), + alpha_(1.0 / (1.0 - theta)), + zetan_(zetan), + eta_((1.0 - std::pow(2.0 / (double)nr, 1.0 - theta)) / + (1.0 - zeta(2, theta) / zetan_)), + threshold_(1.0 + std::pow(0.5, theta)) { + assert(0.0 <= theta); + assert(theta < 1.0); // 1.0 can not be specified. + } + + INLINE size_t operator()() { + double u = rnd_->next() / (double)UINT64_MAX; + double uz = u * zetan_; + if (uz < 1.0) + return 0; + if (uz < threshold_) + return 1; + return (size_t)((double)nr_ * std::pow(eta_ * u - eta_ + 1.0, alpha_)); + } + + uint64_t rand() { return rnd_->next(); } + + static double zeta(size_t nr, double theta) { + double ans = 0.0; + for (size_t i = 0; i < nr; ++i) + ans += std::pow(1.0 / (double)(i + 1), theta); + return ans; + } +}; diff --git a/src_calvin_ext/common/zmq.hpp b/src_calvin_ext/common/zmq.hpp new file mode 100644 index 00000000..9fed64c9 --- /dev/null +++ b/src_calvin_ext/common/zmq.hpp @@ -0,0 +1,193 @@ +/* + Copyright (c) 2007-2011 iMatix Corporation + Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file + + This file is part of 0MQ. + + 0MQ is free software; you can redistribute it and/or modify it under + the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + 0MQ is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program. If not, see . +*/ +// Modified: Shu-Chun Weng + +#ifndef __ZMQ_HPP_INCLUDED__ +#define __ZMQ_HPP_INCLUDED__ + +#include "zmq.h" + +#include +#include +#include + +namespace zmq { + +typedef zmq_free_fn free_fn; +typedef zmq_pollitem_t pollitem_t; + +class error_t : public std::exception { + public: + error_t() : errnum(zmq_errno()) {} + + virtual const char* what() const throw() { return zmq_strerror(errnum); } + + int num() const { return errnum; } + + private: + int errnum; +}; + +inline int poll(zmq_pollitem_t* items_, int nitems_, long timeout_ = -1) { + return zmq_poll(items_, nitems_, timeout_); +} + +inline void device(int device_, void* insocket_, void* outsocket_) { + zmq_device(device_, insocket_, outsocket_); +} + +inline void version(int* major_, int* minor_, int* patch_) { + zmq_version(major_, minor_, patch_); +} + +class message_t : private zmq_msg_t { + friend class socket_t; + + public: + inline message_t() { zmq_msg_init(this); } + + inline message_t(size_t size_) { zmq_msg_init_size(this, size_); } + + inline message_t(void* data_, + size_t size_, + free_fn* ffn_, + void* hint_ = NULL) { + zmq_msg_init_data(this, data_, size_, ffn_, hint_); + } + + inline ~message_t() { + int rc = zmq_msg_close(this); + assert(rc == 0); + } + + inline void rebuild() { + zmq_msg_close(this); + zmq_msg_init(this); + } + + inline void rebuild(size_t size_) { + zmq_msg_close(this); + zmq_msg_init_size(this, size_); + } + + inline void rebuild(void* data_, + size_t size_, + free_fn* ffn_, + void* hint_ = NULL) { + zmq_msg_close(this); + zmq_msg_init_data(this, data_, size_, ffn_, hint_); + } + + inline void move(message_t* msg_) { zmq_msg_move(this, (zmq_msg_t*)msg_); } + + inline void copy(message_t* msg_) { zmq_msg_copy(this, (zmq_msg_t*)msg_); } + + inline void* data() { return zmq_msg_data(this); } + + inline size_t size() { return zmq_msg_size(this); } + + private: + // Disable implicit message copying, so that users won't use shared + // messages (less efficient) without being aware of the fact. + message_t(const message_t&); + void operator=(const message_t&); +}; + +class context_t { + friend class socket_t; + + public: + inline context_t(int io_threads_) { ptr = zmq_init(io_threads_); } + + inline ~context_t() { + int rc = zmq_term(ptr); + assert(rc == 0); + } + + // Be careful with this, it's probably only useful for + // using the C api together with an existing C++ api. + // Normally you should never need to use this. + inline operator void*() { return ptr; } + + private: + void* ptr; + + context_t(const context_t&); + void operator=(const context_t&); +}; + +class socket_t { + public: + inline socket_t(context_t& context_, int type_) { + ptr = zmq_socket(context_.ptr, type_); + } + + inline ~socket_t() { close(); } + + inline operator void*() { return ptr; } + + inline void close() { + if (ptr == NULL) + // already closed + return; + zmq_close(ptr); + ptr = 0; + } + + inline void setsockopt(int option_, const void* optval_, size_t optvallen_) { + zmq_setsockopt(ptr, option_, optval_, optvallen_); + } + + inline void getsockopt(int option_, void* optval_, size_t* optvallen_) { + zmq_getsockopt(ptr, option_, optval_, optvallen_); + } + + inline void bind(const char* addr_) { zmq_bind(ptr, addr_); } + + inline void connect(const char* addr_) { zmq_connect(ptr, addr_); } + + inline bool send(message_t& msg_, int flags_ = 0) { + int rc = zmq_send(ptr, &msg_, flags_); + if (rc == 0) + return true; + if (rc == -1 && zmq_errno() == EAGAIN) + return false; + return false; + } + + inline bool recv(message_t* msg_, int flags_ = 0) { + int rc = zmq_recv(ptr, msg_, flags_); + if (rc == 0) + return true; + if (rc == -1 && zmq_errno() == EAGAIN) + return false; + return false; + } + + private: + void* ptr; + + socket_t(const socket_t&); + void operator=(const socket_t&); +}; + +} // namespace zmq + +#endif diff --git a/src_calvin_3_partitions/deployment/Makefile.inc b/src_calvin_ext/deployment/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/Makefile.inc rename to src_calvin_ext/deployment/Makefile.inc diff --git a/src_calvin_3_partitions/deployment/addhosts.pl b/src_calvin_ext/deployment/addhosts.pl old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/addhosts.pl rename to src_calvin_ext/deployment/addhosts.pl diff --git a/src_calvin_vector_vll/deployment/cluster.cc b/src_calvin_ext/deployment/cluster.cc old mode 100755 new mode 100644 similarity index 84% rename from src_calvin_vector_vll/deployment/cluster.cc rename to src_calvin_ext/deployment/cluster.cc index c04f82b9..9f82aa1e --- a/src_calvin_vector_vll/deployment/cluster.cc +++ b/src_calvin_ext/deployment/cluster.cc @@ -30,8 +30,8 @@ int UpdatePorts(int port_begin, Configuration* config); void Deploy(const Configuration& config, const char* exec); const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; +const char default_port_filename[] = "deployment/portfile"; +const char default_executable_filename[] = "../obj/deployment/db"; const char default_run_config_filename[] = "deploy-run.conf"; const char remote_exec[] = "ssh"; @@ -48,7 +48,6 @@ const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; // sprintf(remote_opt3, remote_opt3_fmt, // cwd, per-exec, node-id, db-args (joined with spaces)) - // TODO(scw): make deployer class; these should be class objs char* cwd; char* db_args; @@ -66,20 +65,19 @@ int main(int argc, char* argv[]) { int arg_begin; const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; + const char* port_file = default_port_filename; + const char* exec = default_executable_filename; for (arg_begin = 1; arg_begin < argc; ++arg_begin) { if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); + printf( + "Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" + " [-d db-exec] [db-args..]\n" + " -c config-file default: %s\n" + " -p port-file default: %s\n" + " -d db-exec default: %s\n", + argv[0], default_input_config_filename, default_port_filename, + default_executable_filename); return 0; } else if (strcmp(argv[arg_begin], "-c") == 0) { config_file = argv[++arg_begin]; @@ -133,14 +131,14 @@ int main(int argc, char* argv[]) { fprintf(port_fp, "%d\n", next_port); fclose(port_fp); -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } + // for (map::iterator it = config.all_nodes.begin(); + // it != config.all_nodes.end(); ++it) { + // char copy_config[1024]; + // snprintf(copy_config, sizeof(copy_config), + // "scp -rp deploy-run.conf %s:db3/deploy-run.conf", + // it->second->host.c_str()); + // system(copy_config); + // } Deploy(config, exec); @@ -181,8 +179,7 @@ int UpdatePorts(int port_begin, Configuration* config) { // Insert host, port_begin> if the host has not appeared. // Otherwise, use the existing host-port pair. map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; + next_port_map.insert(std::make_pair(node->host, port_begin)).first; node->port = port_it->second; port_it->second = port_it->second + 1; @@ -212,14 +209,14 @@ void DeployOne(int nodeID, char remote_opt3[1024]; if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, cwd, + exec, nodeID, db_args); else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, cwd, exec, + nodeID, db_args); else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); + snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, cwd, exec, + nodeID, db_args); // Black magic, don't touch (bug scw if this breaks). int pipefd[2]; @@ -242,10 +239,10 @@ void DeployOne(int nodeID, close(pipefd[1]); } - timespec to_sleep = { 0, 100000000 }; // 0.1 sec + timespec to_sleep = {0, 100000000}; // 0.1 sec nanosleep(&to_sleep, NULL); - (void) config_file; + (void)config_file; } void TerminatingChildren(int sig); @@ -334,7 +331,7 @@ void Deploy(const Configuration& config, const char* exec) { // kill all active components & all ssh procs KillRemote(config, exec, false); - timespec to_sleep = { 1, 0 }; // 1 sec + timespec to_sleep = {1, 0}; // 1 sec nanosleep(&to_sleep, NULL); KillLocal(); } @@ -346,7 +343,8 @@ void TerminatingChildren(int sig) { // try to kill all remote processes spawned void KillRemote(const Configuration& config, - const char* exec, bool client_int) { + const char* exec, + bool client_int) { const char* sig_arg; if (client_int) sig_arg = "-INT"; @@ -354,17 +352,16 @@ void KillRemote(const Configuration& config, sig_arg = "-TERM"; char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); + snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", cwd, exec); - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { + for (map::const_iterator it = config.all_nodes.begin(); + it != config.all_nodes.end(); ++it) { Node* node = it->second; int pid = fork(); if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); + execlp("ssh", "ssh", node->host.c_str(), "killall", sig_arg, + exec_fullpath, NULL); exit(-1); } } diff --git a/src_calvin_ext/deployment/main.cc b/src_calvin_ext/deployment/main.cc new file mode 100644 index 00000000..d4d976ec --- /dev/null +++ b/src_calvin_ext/deployment/main.cc @@ -0,0 +1,249 @@ +// Author: Kun Ren (kun.ren@yale.edu) +// Author: Alexander Thomson (thomson@cs.yale.edu) +// +// Main invokation of a single node in the system. + +#include +#include +#include +#include + +#include "applications/application.h" +#include "applications/microbenchmark.h" +#include "applications/tpcc.h" +#include "applications/ycsb.h" +#include "common/configuration.h" +#include "common/connection.h" +#include "common/definitions.hh" +#include "backend/simple_storage.h" +#include "backend/fetching_storage.h" +#include "backend/collapsed_versioned_storage.h" +#include "scheduler/serial_scheduler.h" +#include "scheduler/deterministic_scheduler.h" +#include "sequencer/sequencer.h" +#include "proto/tpcc_args.pb.h" +#include "proto/txn.pb.h" + +// グローバル変数 +map latest_order_id_for_customer; +map latest_order_id_for_district; +map smallest_order_id_for_district; +map customer_for_order; +unordered_map next_order_id_for_district; +map item_for_order_line; +map order_line_number; +vector *involed_customers; + +pthread_mutex_t mutex_; +pthread_mutex_t mutex_for_item; + +// Microbenchmark用クライアント +class MClient : public Client +{ +public: + // --- MODIFICATION START --- + // コンストラクタの第3引数の型から const を外す + MClient(Configuration *config, int mp, Application *app) + // microbenchmark_ を const ではないポインタとして初期化 + : microbenchmark_(static_cast(app)), config_(config), + percent_mp_(mp) + { + } + // --- MODIFICATION END --- + + virtual ~MClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + if (config_->all_nodes.size() > 1 && (rand() % 100) < percent_mp_) + { + int other; + do + { + other = rand() % config_->all_nodes.size(); + } while (other == config_->this_node_id); + // これでエラーなく呼び出せる + *txn = microbenchmark_->MicroTxnMP(txn_id, config_->this_node_id, other); + } + else + { + *txn = microbenchmark_->MicroTxnSP(txn_id, config_->this_node_id); + } + } + +private: + // --- MODIFICATION START --- + // メンバ変数の型から const を外す + Microbenchmark *microbenchmark_; + // --- MODIFICATION END --- + Configuration *config_; + int percent_mp_; +}; + +// TPCC用クライアント +class TClient : public Client +{ +public: + TClient(Configuration *config, int mp, Application *app) + : config_(config), percent_mp_(mp), tpcc_(static_cast(app)) {} + virtual ~TClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + TPCCArgs args; + args.set_system_time(GetTime()); + args.set_multipartition((rand() % 100) < percent_mp_); + + string args_string; + args.SerializeToString(&args_string); + + int random_txn_type = rand() % 100; + if (random_txn_type < 45) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); + } + else if (random_txn_type < 88) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); + } + else if (random_txn_type < 92) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); + } + else if (random_txn_type < 96) + { + *txn = tpcc_->NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); + } + else + { + *txn = tpcc_->NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); + } + } + +private: + Configuration *config_; + int percent_mp_; + TPCC *tpcc_; +}; + +// YCSB用クライアント +class YClient : public Client +{ +public: + YClient(Configuration *config, int mp, Application *ycsb_app) + : config_(config), percent_mp_(mp), ycsb_app_(ycsb_app) {} + virtual ~YClient() {} + virtual void GetTxn(TxnProto **txn, int txn_id) + { + string args_string = ""; + *txn = ycsb_app_->NewTxn(txn_id, 0, args_string, config_); + } + +private: + Configuration *config_; + int percent_mp_; + Application *ycsb_app_; +}; + +// シグナルハンドラ +void stop(int sig) +{ + exit(sig); +} + +int main(int argc, char **argv) +{ + // 引数チェック + if (argc < 4) + { + fprintf(stderr, "Usage: %s [f for fetching]\n", + argv[0]); + exit(1); + } + bool useFetching = (argc > 4 && argv[4][0] == 'f'); + + signal(SIGINT, &stop); + signal(SIGTERM, &stop); + + // 設定オブジェクトの構築 + Configuration config(StringToInt(argv[1]), "deploy-run.conf"); + + // 通信マルチプレクサの構築と起動 + ConnectionMultiplexer multiplexer(&config); + + // ベンチマークの種類に応じてApplicationオブジェクトを生成 + Application *application; + if (argv[2][0] == 'm') + { + application = new Microbenchmark(config.all_nodes.size(), HOT); + } + else if (argv[2][0] == 't') + { + application = new TPCC(); + } + else if (argv[2][0] == 'y') + { + // YCSBのパラメータをここで自由に設定 + const double read_ratio = 1.0; + const double skew = 0.99; + const uint64 db_size = DB_SIZE; + const uint64 hot_records = 10; // この値はdefinitions.hhに移動するとより良い + application = new YCSB(read_ratio, skew, db_size, hot_records); + } + else + { + fprintf(stderr, "Unknown benchmark type: %s\n", argv[2]); + exit(1); + } + + // Applicationオブジェクトに対応するClientオブジェクトを生成 + Client *client; + if (argv[2][0] == 'm') + { + client = new MClient(&config, atoi(argv[3]), application); + } + else if (argv[2][0] == 't') + { + client = new TClient(&config, atoi(argv[3]), application); + } + else // 'y' + { + client = new YClient(&config, atoi(argv[3]), application); + } + + pthread_mutex_init(&mutex_, NULL); + pthread_mutex_init(&mutex_for_item, NULL); + involed_customers = new vector; + + // ストレージ層の選択と初期化 + Storage *storage; + if (!useFetching) + { + storage = new SimpleStorage(); + } + else + { + storage = FetchingStorage::BuildStorage(); + } + storage->Initmutex(); + application->InitializeStorage(storage, &config); + + // シーケンサコンポーネントの初期化と起動 + Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, + storage); + + // スケジューラをメインスレッドで実行 + DeterministicScheduler scheduler(&config, + multiplexer.NewConnection("scheduler_"), + storage, + application); + + // 180秒間実行 + Spin(180); + + // メモリ解放 + delete client; + delete application; + delete storage; + delete involed_customers; + + return 0; +} \ No newline at end of file diff --git a/src_calvin_3_partitions/deployment/makeconf.pl b/src_calvin_ext/deployment/makeconf.pl old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/makeconf.pl rename to src_calvin_ext/deployment/makeconf.pl diff --git a/src_calvin_3_partitions/deployment/portfile b/src_calvin_ext/deployment/portfile old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/portfile rename to src_calvin_ext/deployment/portfile diff --git a/src_calvin_3_partitions/deployment/run.pbs b/src_calvin_ext/deployment/run.pbs old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/run.pbs rename to src_calvin_ext/deployment/run.pbs diff --git a/src_calvin_3_partitions/deployment/test.conf b/src_calvin_ext/deployment/test.conf old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/deployment/test.conf rename to src_calvin_ext/deployment/test.conf diff --git a/src_calvin_3_partitions/paxos/Makefile.inc b/src_calvin_ext/paxos/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/paxos/Makefile.inc rename to src_calvin_ext/paxos/Makefile.inc diff --git a/src_calvin_vector_vll/paxos/paxos.cc b/src_calvin_ext/paxos/paxos.cc old mode 100755 new mode 100644 similarity index 76% rename from src_calvin_vector_vll/paxos/paxos.cc rename to src_calvin_ext/paxos/paxos.cc index d0523e10..17e7d624 --- a/src_calvin_vector_vll/paxos/paxos.cc +++ b/src_calvin_ext/paxos/paxos.cc @@ -21,15 +21,15 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { if (s.substr(0, 10) == "clientPort") { int pos1 = s.find('='); int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); + port = s.substr(pos1 + 1, pos2 - pos1 - 1); } else if (s.substr(0, 7) == "server1") { int pos1 = s.find('='); int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); + ip = s.substr(pos1 + 1, pos2 - pos1 - 1); } else if (s.substr(0, 7) == "timeout") { int pos1 = s.find('='); int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); + timeout = s.substr(pos1 + 1, pos2 - pos1 - 1); } } connection_string = ip + ":" + port; @@ -42,8 +42,8 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { } // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); + zh_ = zookeeper_init(connection_string.c_str(), NULL, atoi(timeout.c_str()), + 0, NULL, 0); if (zh_ == NULL) { printf("Connection to zookeeper failed.\n"); return; @@ -56,9 +56,8 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { // If multiple nodes executing this code to both see that // the root doesn't exist, only one node creates /root // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); + int create_rc = + zoo_create(zh_, "/root", NULL, 0, &ZOO_OPEN_ACL_UNSAFE, 0, NULL, 0); if (create_rc != ZOK && create_rc != ZNODEEXISTS) { printf("zoo_create error:error number is %d\n", create_rc); } @@ -70,10 +69,9 @@ Paxos::Paxos(const string& zookeeper_config_file, bool reader) { char current_read_batch_path[23]; snprintf(current_read_batch_path, sizeof(current_read_batch_path), "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); + int get_rc = zoo_aget( + zh_, current_read_batch_path, 0, get_data_completion, + reinterpret_cast(new pair(i, this))); if (get_rc) { printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); } @@ -97,8 +95,7 @@ void Paxos::SubmitBatch(const string& batch_data) { // Submit batch means that create new znode below the root directory. int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); + ZOO_SEQUENCE | ZOO_EPHEMERAL, acreate_completion, NULL); if (rc != ZOK) { printf("zoo_acreate error:error number is %d\n", rc); } @@ -127,8 +124,11 @@ void Paxos::GetNextBatchBlocking(string* batch_data) { } } -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { +void Paxos::get_data_completion(int rc, + const char* value, + int value_len, + const struct Stat* stat, + const void* data) { // XXX(scw): using const_cast is disgusting pair* previous_data = reinterpret_cast*>(const_cast(data)); @@ -142,34 +142,34 @@ void Paxos::get_data_completion(int rc, const char *value, int value_len, // Set the number of batch which will be got from zookeeper next time // (just plus the CONCURRENT_GETS). next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); + pthread_mutex_lock( + &paxos->mutexes_[previous_index_for_aget % CONCURRENT_GETS]); paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); + pthread_mutex_unlock( + &paxos->mutexes_[previous_index_for_aget % CONCURRENT_GETS]); // If there are no new batch in the zookeeper, just wait for a while // and continue to get from zookeeper. } else if (rc == ZNONODE) { next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); + usleep(0.2 * 1000); } else { return; } // Continue to get a batch from zookeeper. char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); + snprintf(current_read_batch_path, sizeof(current_read_batch_path), "%s%010lu", + "/root/batch-", next_index_for_aget); previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); + int get_rc = + zoo_aget(paxos->zh_, current_read_batch_path, 0, get_data_completion, + reinterpret_cast(previous_data)); if (get_rc) { return; } } -void Paxos::acreate_completion(int rc, const char *name, const void * data) { +void Paxos::acreate_completion(int rc, const char* name, const void* data) { if (rc) { printf("Error %d for zoo_acreate.\n", rc); } @@ -187,10 +187,10 @@ void StartZookeeper(const string& zookeeper_config_file) { if (line.substr(0, 6) == "server") { int pos1 = line.find('='); int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); + zookeepers.push_back(line.substr(pos1 + 1, pos2 - pos1 - 1)); } } - for (unsigned int i = 0; i< zookeepers.size(); i++) { + for (unsigned int i = 0; i < zookeepers.size(); i++) { // Generate the ssh command. string ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + @@ -205,8 +205,8 @@ void StartZookeeper(const string& zookeeper_config_file) { // This function will automatically stop zookeeper server based on the // zookeeper config file(generate ssh commands and execute them). void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; + vector zookeepers; + string line, port, ssh_command; // Read zookeeper config file. ifstream in(zookeeper_config_file.c_str()); // Put all zookeeper server's ip into the vector. @@ -214,24 +214,23 @@ void StopZookeeper(const string& zookeeper_config_file) { if (line.substr(0, 6) == "server") { int pos1 = line.find('='); int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); + zookeepers.push_back(line.substr(pos1 + 1, pos2 - pos1 - 1)); } if (line.substr(0, 10) == "clientPort") { int pos1 = line.find('='); int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); + port = line.substr(pos1 + 1, pos2 - pos1 - 1); } } ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; + " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " + + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; system(ssh_command.c_str()); sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { + for (unsigned int i = 0; i < zookeepers.size(); i++) { // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; + ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" + + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; system(ssh_command.c_str()); } } - diff --git a/src_calvin_vector_vll/paxos/paxos.h b/src_calvin_ext/paxos/paxos.h old mode 100755 new mode 100644 similarity index 88% rename from src_calvin_vector_vll/paxos/paxos.h rename to src_calvin_ext/paxos/paxos.h index bbcd711d..f761ab25 --- a/src_calvin_vector_vll/paxos/paxos.h +++ b/src_calvin_ext/paxos/paxos.h @@ -53,7 +53,7 @@ class Paxos { private: // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; + zhandle_t* zh_; // Record the serial number of the batch which will be read next time. uint64 next_read_batch_index_; @@ -68,11 +68,14 @@ class Paxos { // For zoo_aget completion function, this method will be invoked // at the end of a asynchronous call( zoo_aget is asynchronous call // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); + static void get_data_completion(int rc, + const char* value, + int value_len, + const struct Stat* stat, + const void* data); // For zoo_acreate completion function, this method will be invoked // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); + static void acreate_completion(int rc, const char* name, const void* data); }; #endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_calvin_3_partitions/paxos/zookeeper.conf b/src_calvin_ext/paxos/zookeeper.conf old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/paxos/zookeeper.conf rename to src_calvin_ext/paxos/zookeeper.conf diff --git a/src_calvin_3_partitions/proto/Makefile.inc b/src_calvin_ext/proto/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/proto/Makefile.inc rename to src_calvin_ext/proto/Makefile.inc diff --git a/src_calvin_vector_vll/proto/message.proto b/src_calvin_ext/proto/message.proto old mode 100755 new mode 100644 similarity index 98% rename from src_calvin_vector_vll/proto/message.proto rename to src_calvin_ext/proto/message.proto index 8b43a929..af3cb66a --- a/src_calvin_vector_vll/proto/message.proto +++ b/src_calvin_ext/proto/message.proto @@ -43,6 +43,7 @@ message MessageProto { // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn // batch being sent. optional int64 batch_number = 21; + optional bool batch_has_rw = 120; // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and // result of a read, respectively. diff --git a/src_calvin_3_partitions/proto/tpcc.proto b/src_calvin_ext/proto/tpcc.proto old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/proto/tpcc.proto rename to src_calvin_ext/proto/tpcc.proto diff --git a/src_calvin_3_partitions/proto/tpcc_args.proto b/src_calvin_ext/proto/tpcc_args.proto old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/proto/tpcc_args.proto rename to src_calvin_ext/proto/tpcc_args.proto diff --git a/src_calvin_3_partitions/proto/txn.proto b/src_calvin_ext/proto/txn.proto old mode 100755 new mode 100644 similarity index 98% rename from src_calvin_3_partitions/proto/txn.proto rename to src_calvin_ext/proto/txn.proto index 464a8719..e70d5555 --- a/src_calvin_3_partitions/proto/txn.proto +++ b/src_calvin_ext/proto/txn.proto @@ -53,5 +53,6 @@ message TxnProto { // Node ids of nodes that participate as readers and writers in this txn. repeated int32 readers = 40; repeated int32 writers = 41; + optional bool read_only = 43; } diff --git a/src_calvin_3_partitions/scheduler/Makefile.inc b/src_calvin_ext/scheduler/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/scheduler/Makefile.inc rename to src_calvin_ext/scheduler/Makefile.inc diff --git a/src_calvin_vector_vll/scheduler/deterministic_lock_manager.cc b/src_calvin_ext/scheduler/deterministic_lock_manager.cc similarity index 84% rename from src_calvin_vector_vll/scheduler/deterministic_lock_manager.cc rename to src_calvin_ext/scheduler/deterministic_lock_manager.cc index 15d002e3..7741cee8 100644 --- a/src_calvin_vector_vll/scheduler/deterministic_lock_manager.cc +++ b/src_calvin_ext/scheduler/deterministic_lock_manager.cc @@ -12,12 +12,10 @@ using std::vector; -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) +DeterministicLockManager::DeterministicLockManager(deque* ready_txns, + Configuration* config) + : configuration_(config), ready_txns_(ready_txns) { + for (int i = 0; i < LOCK_TABLE_SIZE; i++) lock_table_[i] = new deque(); } @@ -29,10 +27,11 @@ int DeterministicLockManager::Lock(TxnProto* txn) { // Only lock local keys. if (IsLocal(txn->read_write_set(i))) { deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - + deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { + for (it = key_requests->begin(); + it != key_requests->end() && it->key != txn->read_write_set(i); + ++it) { } deque* requests; if (it == key_requests->end()) { @@ -41,7 +40,7 @@ int DeterministicLockManager::Lock(TxnProto* txn) { } else { requests = it->locksrequest; } - + // Only need to request this if lock txn hasn't already requested it. if (requests->empty() || txn != requests->back().txn) { requests->push_back(LockRequest(WRITE, txn)); @@ -58,10 +57,10 @@ int DeterministicLockManager::Lock(TxnProto* txn) { // Only lock local keys. if (IsLocal(txn->read_set(i))) { deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - + deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { + for (it = key_requests->begin(); + it != key_requests->end() && it->key != txn->read_set(i); ++it) { } deque* requests; if (it == key_requests->end()) { @@ -70,7 +69,7 @@ int DeterministicLockManager::Lock(TxnProto* txn) { } else { requests = it->locksrequest; } - + // Only need to request this if lock txn hasn't already requested it. if (requests->empty() || txn != requests->back().txn) { requests->push_back(LockRequest(READ, txn)); @@ -100,9 +99,9 @@ void DeterministicLockManager::Release(TxnProto* txn) { Release(txn->read_set(i), txn); // Currently commented out because nothing in any write set can conflict // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); + // for (int i = 0; i < txn->write_set_size(); i++) + // if (IsLocal(txn->write_set(i))) + // Release(txn->write_set(i), txn); for (int i = 0; i < txn->read_write_set_size(); i++) if (IsLocal(txn->read_write_set(i))) Release(txn->read_write_set(i), txn); @@ -110,21 +109,19 @@ void DeterministicLockManager::Release(TxnProto* txn) { void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - + deque* key_requests = lock_table_[Hash(key)]; + deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { + for (it1 = key_requests->begin(); + it1 != key_requests->end() && it1->key != key; ++it1) { } deque* requests = it1->locksrequest; - // Seek to the target request. Note whether any write lock requests precede // the target. bool write_requests_precede_target = false; deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { + for (it = requests->begin(); it != requests->end() && it->txn != txn; ++it) { if (it->mode == WRITE) write_requests_precede_target = true; } @@ -154,8 +151,8 @@ void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { // If a sequence of read lock requests follows, grant all of them. for (; it != requests->end() && it->mode == READ; ++it) new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) + } else if (!write_requests_precede_target && target->mode == WRITE && + it->mode == READ) { // (c) // If a sequence of read lock requests follows, grant all of them. for (; it != requests->end() && it->mode == READ; ++it) new_owners.push_back(it->txn); @@ -177,9 +174,7 @@ void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { requests->erase(target); if (requests->size() == 0) { delete requests; - key_requests->erase(it1); + key_requests->erase(it1); } - } } - diff --git a/src_calvin_3_partitions/scheduler/deterministic_lock_manager.h b/src_calvin_ext/scheduler/deterministic_lock_manager.h old mode 100755 new mode 100644 similarity index 91% rename from src_calvin_3_partitions/scheduler/deterministic_lock_manager.h rename to src_calvin_ext/scheduler/deterministic_lock_manager.h index 1bba5425..6416e90d --- a/src_calvin_3_partitions/scheduler/deterministic_lock_manager.h +++ b/src_calvin_ext/scheduler/deterministic_lock_manager.h @@ -13,18 +13,18 @@ #include "common/configuration.h" #include "scheduler/lock_manager.h" #include "common/utils.h" +#include "common/definitions.hh" -using std::tr1::unordered_map; using std::deque; +using std::tr1::unordered_map; -#define TABLE_SIZE 1000000 +// #define LOCK_TABLE_SIZE 1000000 class TxnProto; class DeterministicLockManager { public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); + DeterministicLockManager(deque* ready_txns, Configuration* config); virtual ~DeterministicLockManager() {} virtual int Lock(TxnProto* txn); virtual void Release(const Key& key, TxnProto* txn); @@ -37,7 +37,7 @@ class DeterministicLockManager { hash = hash ^ (key[i]); hash = hash * 16777619; } - return hash % TABLE_SIZE; + return hash % LOCK_TABLE_SIZE; } bool IsLocal(const Key& key) { @@ -60,15 +60,14 @@ class DeterministicLockManager { TxnProto* txn; // Pointer to txn requesting the lock. LockMode mode; // Specifies whether this is a read or write lock request. }; - + struct KeysList { KeysList(Key m, deque* t) : key(m), locksrequest(t) {} Key key; deque* locksrequest; }; - - deque* lock_table_[TABLE_SIZE]; + deque* lock_table_[LOCK_TABLE_SIZE]; // Queue of pointers to transactions that have acquired all locks that // they have requested. 'ready_txns_[key].front()' is the owner of the lock diff --git a/src_calvin_4_partitions/scheduler/deterministic_scheduler.cc b/src_calvin_ext/scheduler/deterministic_scheduler.cc old mode 100755 new mode 100644 similarity index 57% rename from src_calvin_4_partitions/scheduler/deterministic_scheduler.cc rename to src_calvin_ext/scheduler/deterministic_scheduler.cc index fb49b927..71b1ea22 --- a/src_calvin_4_partitions/scheduler/deterministic_scheduler.cc +++ b/src_calvin_ext/scheduler/deterministic_scheduler.cc @@ -1,4 +1,4 @@ -// Author: Kun Ren (kun.ren@yale.edu) +// Author: Kun Ren (kun@cs.yale.edu) // Author: Alexander Thomson (thomson@cs.yale.edu) // // The deterministic lock manager implements deterministic locking as described @@ -24,6 +24,7 @@ #include "common/utils.h" #include "common/zmq.hpp" #include "common/connection.h" +#include "common/definitions.hh" #include "backend/storage.h" #include "backend/storage_manager.h" #include "proto/message.pb.h" @@ -35,13 +36,17 @@ // to get COLD_CUTOFF #include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST +#include "common/debug.hh" + +using std::map; using std::pair; using std::string; using std::tr1::unordered_map; using zmq::socket_t; -using std::map; -static void DeleteTxnPtr(void* data, void* hint) { free(data); } +static void DeleteTxnPtr(void* data, void* hint) { + free(data); +} void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); @@ -62,53 +67,55 @@ DeterministicScheduler::DeterministicScheduler(Configuration* conf, Connection* batch_connection, Storage* storage, const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); + : configuration_(conf), + batch_connection_(batch_connection), + storage_(storage), + application_(application) { + ready_txns_ = new std::deque(); lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - + txns_queue = new AtomicQueue(); done_queue = new AtomicQueue(); + ro_counter_ = 0; + for (int i = 0; i < NUM_WORKERS; i++) { + ro_queues[i] = new AtomicQueue(); + } - for (int i = 0; i < NUM_THREADS; i++) { + for (int i = 0; i < NUM_WORKERS; i++) { message_queues[i] = new AtomicQueue(); } -Spin(2); + Spin(1); // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; + cpu_set_t cpuset; + pthread_attr_t attr1; pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - + // pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); + CPU_ZERO(&cpuset); - CPU_SET(7, &cpuset); + CPU_SET(LOCK_MANAGER_CORE, &cpuset); pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, reinterpret_cast(this)); - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { + for (int i = 0; i < NUM_WORKERS; i++) { string channel("scheduler"); channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); + thread_connections_[i] = batch_connection_->multiplexer()->NewConnection( + channel, &message_queues[i]); pthread_attr_t attr; pthread_attr_init(&attr); CPU_ZERO(&cpuset); - if (i == 0 || i == 1) - CPU_SET(i, &cpuset); - else - CPU_SET(i+2, &cpuset); + CPU_SET(GET_WORKER_CORE(i), &cpuset); pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); pthread_create(&(threads_[i]), &attr, RunWorkerThread, reinterpret_cast( - new pair(i, this))); + new pair(i, this))); } - } void UnfetchAll(Storage* storage, TxnProto* txn) { @@ -131,6 +138,8 @@ void* DeterministicScheduler::RunWorkerThread(void* arg) { unordered_map active_txns; + PrintCpu("Worker", thread); + // Begin main loop. MessageProto message; while (true) { @@ -146,47 +155,71 @@ void* DeterministicScheduler::RunWorkerThread(void* arg) { scheduler->application_->Execute(txn, manager); delete manager; - scheduler->thread_connections_[thread]-> - UnlinkChannel(IntToString(txn->txn_id())); + scheduler->thread_connections_[thread]->UnlinkChannel( + IntToString(txn->txn_id())); active_txns.erase(message.destination_channel()); - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); + + // 完了トランザクションを done_queue へ(RO/RW 共通) scheduler->done_queue->Push(txn); } } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { + // No remote read result found, try to get a new transaction. + + // 1) まず RW キュー (txns_queue) をチェック + TxnProto* txn; // RW txn + bool got_rw = scheduler->txns_queue->Pop(&txn); + if (got_rw == true) { // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); + StorageManager* manager = new StorageManager( + scheduler->configuration_, scheduler->thread_connections_[thread], + scheduler->storage_, txn); + + if (manager->ReadyToExecute()) { + // No remote reads. Execute and clean up. + scheduler->application_->Execute(txn, manager); + delete manager; + + // 完了したRW Txnをdone_queueに送る + scheduler->done_queue->Push(txn); + } else { + scheduler->thread_connections_[thread]->LinkChannel( + IntToString(txn->txn_id())); + // There are outstanding remote reads. + active_txns[IntToString(txn->txn_id())] = manager; + } + + } else { + // 2) RW が無ければ、RO キューをチェック + TxnProto* ro_txn; + bool got_ro = scheduler->ro_queues[thread]->Pop(&ro_txn); + if (got_ro == true) { + // Got a Read-Only txn. + StorageManager* manager = new StorageManager( + scheduler->configuration_, scheduler->thread_connections_[thread], + scheduler->storage_, ro_txn); - // Writes occur at this node. if (manager->ReadyToExecute()) { // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); + scheduler->application_->Execute(ro_txn, manager); delete manager; - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); + // 完了したRO Txnをdone_queueに送る + scheduler->done_queue->Push(ro_txn); } else { - scheduler->thread_connections_[thread]-> - LinkChannel(IntToString(txn->txn_id())); - // There are outstanding remote reads. - active_txns[IntToString(txn->txn_id())] = manager; + // RO txn has outstanding remote reads. + scheduler->thread_connections_[thread]->LinkChannel( + IntToString(ro_txn->txn_id())); + active_txns[IntToString(ro_txn->txn_id())] = manager; } + } } } } return NULL; } -DeterministicScheduler::~DeterministicScheduler() { -} + +DeterministicScheduler::~DeterministicScheduler() {} // Returns ptr to heap-allocated unordered_map batches; @@ -213,47 +246,65 @@ MessageProto* GetBatch(int batch_id, Connection* connection) { } void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); + PrintCpu("Lock Manager", 0); + + DeterministicScheduler* scheduler = + reinterpret_cast(arg); // Run main loop. MessageProto message; MessageProto* batch_message = NULL; - int txns = 0; + int txns = 0; // <--- これがシステム全体のスループットカウンターになる double time = GetTime(); - int executing_txns = 0; + int executing_txns = 0; // <--- これはRWロック待ちのTxnのみをカウント int pending_txns = 0; int batch_offset = 0; int batch_number = 0; -//int test = 0; + // int test = 0; + + int tasks[Task::Size] = {0}; + + std::string task_names[Task::Size] = {"ProcessDoneTransaction", + "LoadNextBatch", "AdvanceBatch", + "Locking", "ProcessReadyTransaction"}; + + // ... (コメントアウトされた goto 部分は省略) ... + while (true) { TxnProto* done_txn; bool got_it = scheduler->done_queue->Pop(&done_txn); if (got_it == true) { - // We have received a finished transaction back, release the lock - scheduler->lock_manager_->Release(done_txn); - executing_txns--; - - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) - txns++; - + // -------- [ここから変更] -------- + + // 1. 完了したトランザクションをカウント (RO/RW両方) + txns++; + + // 2. RWトランザクションの場合のみ、ロックを解放 + if (done_txn->read_only() == false) { + scheduler->lock_manager_->Release(done_txn); + executing_txns--; + } + + // 3. トランザクションオブジェクトを削除 delete done_txn; + + // -------- [ここまで変更] -------- } else { // Have we run out of txns in our batch? Let's get some new ones. if (batch_message == NULL) { batch_message = GetBatch(batch_number, scheduler->batch_connection_); - // Done with current batch, get next. + // Done with current batch, get next. } else if (batch_offset >= batch_message->data_size()) { batch_offset = 0; batch_number++; delete batch_message; batch_message = GetBatch(batch_number, scheduler->batch_connection_); - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { + // Current batch has remaining txns, grab up to 10. + } else if (executing_txns + pending_txns < MAX_ACTIVE_TXNS) { + for (int i = 0; i < LOCK_BATCH_SIZE; i++) { if (batch_offset >= batch_message->data_size()) { // Oops we ran out of txns in this batch. Stop adding txns for now. break; @@ -262,37 +313,53 @@ void* DeterministicScheduler::LockManagerThread(void* arg) { txn->ParseFromString(batch_message->data(batch_offset)); batch_offset++; - scheduler->lock_manager_->Lock(txn); - pending_txns++; + if (txn->read_only()) { + // Read-Only トランザクション + int worker_id = (scheduler->ro_counter_++) % NUM_WORKERS; + scheduler->ro_queues[worker_id]->Push(txn); + } else { + // Read-Write トランザクション + scheduler->lock_manager_->Lock(txn); + pending_txns++; + } } - } } - // Start executing any and all ready transactions to get them off our plate + // Start executing any and all ready (RW) transactions to get them off our plate while (!scheduler->ready_txns_->empty()) { TxnProto* txn = scheduler->ready_txns_->front(); scheduler->ready_txns_->pop_front(); pending_txns--; executing_txns++; - scheduler->txns_queue->Push(txn); - //scheduler->SendTxnPtr(scheduler->requests_out_, txn); + scheduler->txns_queue->Push(txn); } // Report throughput. if (GetTime() > time + 1) { double total_time = GetTime() - time; + + std::string task_output = "Tasks: "; + for (int i = 0; i < Task::Size; i++) { + task_output.append(task_names[i] + ": " + std::to_string(tasks[i]) + + ", "); + } + + // 'txns' が正確な合計スループットを反映するようになった std::cout << "Completed " << (static_cast(txns) / total_time) << " txns/sec, " - //<< test<< " for drop speed , " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; + //<< test<< " for drop speed , " + << executing_txns << " executing (RW), " << pending_txns + << " pending (RW), " << "\n" + << task_output << "\n" + << std::flush; // Reset txn count. time = GetTime(); txns = 0; - //test ++; + // test ++; + memset(tasks, 0, sizeof(tasks)); } } return NULL; -} +} \ No newline at end of file diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.h b/src_calvin_ext/scheduler/deterministic_scheduler.h old mode 100755 new mode 100644 similarity index 62% rename from src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.h rename to src_calvin_ext/scheduler/deterministic_scheduler.h index 9ab327f4..d71cc68a --- a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.h +++ b/src_calvin_ext/scheduler/deterministic_scheduler.h @@ -7,87 +7,92 @@ // transaction in the specified order may acquire any locks. Each lock is then // granted to transactions in the order in which they requested them (i.e. in // the global transaction order). - #ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ #define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - #include - #include - #include "scheduler/scheduler.h" -#include "common/utils.h" +#include "common/utils.h" // <-- [.hh から .h に修正] +#include "common/definitions.hh" #include "proto/txn.pb.h" #include "proto/message.pb.h" -using std::deque; + +using std::deque; namespace zmq { class socket_t; class message_t; -} +} // namespace zmq using zmq::socket_t; - class Configuration; class Connection; class DeterministicLockManager; class Storage; class TxnProto; - -#define NUM_THREADS 4 -// #define PREFETCHING - class DeterministicScheduler : public Scheduler { public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); + enum Task { + ProcessDoneTransaction, + LoadNextBatch, + AdvanceBatch, + Locking, + ProcessReadyTransaction, + Size + }; + DeterministicScheduler(Configuration* conf, + Connection* batch_connection, + Storage* storage, + const Application* application); virtual ~DeterministicScheduler(); - private: // Function for starting main loops in a separate pthreads. static void* RunWorkerThread(void* arg); - static void* LockManagerThread(void* arg); - void SendTxnPtr(socket_t* socket, TxnProto* txn); TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - // Configuration specifying node & system settings. Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; - + pthread_t lock_manager_thread_; // Connection for receiving txn batches from sequencer. Connection* batch_connection_; - // Storage layer used in application execution. Storage* storage_; - // Application currently being run. const Application* application_; - // The per-node lock manager tracks what transactions have temporary ownership // of what database objects, allowing the scheduler to track LOCAL conflicts // and enforce equivalence to transaction orders. DeterministicLockManager* lock_manager_; - // Queue of transaction ids of transactions that have acquired all locks that // they have requested. std::deque* ready_txns_; - // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; + // socket_t* requests_out_; + // socket_t* requests_in_; + // socket_t* responses_out_[NUM_WORKERS]; + // socket_t* responses_in_; + // RWトランザクション用のキュー (ロックマネージャ -> ワーカー) AtomicQueue* txns_queue; + // 完了したRWトランザクション用のキュー (ワーカー -> ロックマネージャ) AtomicQueue* done_queue; - AtomicQueue* message_queues[NUM_THREADS]; + // -------- [ここへ移動] -------- + // ROキュー振り分け用のラウンドロビンカウンター + uint64_t ro_counter_; + // -------- [ここまで] -------- + + // --- 配列メンバはクラス定義の最後にまとめる --- + // Thread contexts and their associated Connection objects. + pthread_t threads_[NUM_WORKERS]; + Connection* thread_connections_[NUM_WORKERS]; + + AtomicQueue* message_queues[NUM_WORKERS]; + // ROトランザクション用のキュー (ロックマネージャ -> ワーカー) + AtomicQueue* ro_queues[NUM_WORKERS]; + }; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ +#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ \ No newline at end of file diff --git a/src_calvin_3_partitions/scheduler/lock_manager.h b/src_calvin_ext/scheduler/lock_manager.h old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/scheduler/lock_manager.h rename to src_calvin_ext/scheduler/lock_manager.h diff --git a/src_calvin_vector_vll/scheduler/scheduler.h b/src_calvin_ext/scheduler/scheduler.h old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_vector_vll/scheduler/scheduler.h rename to src_calvin_ext/scheduler/scheduler.h diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.cc b/src_calvin_ext/scheduler/serial_scheduler.cc old mode 100755 new mode 100644 similarity index 88% rename from src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.cc rename to src_calvin_ext/scheduler/serial_scheduler.cc index 1d2c9bfe..6b6469c8 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.cc +++ b/src_calvin_ext/scheduler/serial_scheduler.cc @@ -18,11 +18,14 @@ #include "proto/message.pb.h" #include "proto/txn.pb.h" -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} +SerialScheduler::SerialScheduler(Configuration* conf, + Connection* connection, + Storage* storage, + bool checkpointing) + : configuration_(conf), + connection_(connection), + storage_(storage), + checkpointing_(checkpointing) {} SerialScheduler::~SerialScheduler() {} diff --git a/src_calvin_vector_vll/scheduler/serial_scheduler.h b/src_calvin_ext/scheduler/serial_scheduler.h old mode 100755 new mode 100644 similarity index 84% rename from src_calvin_vector_vll/scheduler/serial_scheduler.h rename to src_calvin_ext/scheduler/serial_scheduler.h index f99958c6..0fc5e328 --- a/src_calvin_vector_vll/scheduler/serial_scheduler.h +++ b/src_calvin_ext/scheduler/serial_scheduler.h @@ -14,8 +14,10 @@ class Storage; class SerialScheduler : public Scheduler { public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); + SerialScheduler(Configuration* conf, + Connection* connection, + Storage* storage, + bool checkpointing); virtual ~SerialScheduler(); virtual void Run(const Application& application); diff --git a/src_calvin_3_partitions/sequencer/Makefile.inc b/src_calvin_ext/sequencer/Makefile.inc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/sequencer/Makefile.inc rename to src_calvin_ext/sequencer/Makefile.inc diff --git a/src_calvin_4_partitions/sequencer/sequencer.cc b/src_calvin_ext/sequencer/sequencer.cc old mode 100755 new mode 100644 similarity index 76% rename from src_calvin_4_partitions/sequencer/sequencer.cc rename to src_calvin_ext/sequencer/sequencer.cc index d0eb3b77..63c488ba --- a/src_calvin_4_partitions/sequencer/sequencer.cc +++ b/src_calvin_ext/sequencer/sequencer.cc @@ -18,16 +18,17 @@ #include "common/configuration.h" #include "common/connection.h" #include "common/utils.h" +#include "common/debug.hh" #include "proto/message.pb.h" #include "proto/txn.pb.h" #ifdef PAXOS -# include "paxos/paxos.h" +#include "paxos/paxos.h" #endif using std::map; using std::multimap; -using std::set; using std::queue; +using std::set; #ifdef LATENCY_TEST double sequencer_recv[SAMPLES]; @@ -41,51 +42,49 @@ double worker_end[SAMPLES]; double scheduler_unlock[SAMPLES]; #endif -void* Sequencer::RunSequencerWriter(void *arg) { +void* Sequencer::RunSequencerWriter(void* arg) { reinterpret_cast(arg)->RunWriter(); return NULL; } -void* Sequencer::RunSequencerReader(void *arg) { +void* Sequencer::RunSequencerReader(void* arg) { reinterpret_cast(arg)->RunReader(); return NULL; } -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), storage_(storage), deconstructor_invoked_(false) { +Sequencer::Sequencer(Configuration* conf, + Connection* connection, + Client* client, + Storage* storage) + : epoch_duration_(EPOCH_DURATION), + configuration_(conf), + connection_(connection), + client_(client), + storage_(storage), + deconstructor_invoked_(false) { pthread_mutex_init(&mutex_, NULL); // Start Sequencer main loops running in background thread. -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - + cpu_set_t cpuset; + pthread_attr_t attr_writer; + pthread_attr_init(&attr_writer); + // pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + CPU_ZERO(&cpuset); + CPU_SET(SEQUENCER_WRITER_CORE, &cpuset); + pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); + reinterpret_cast(this)); -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); + CPU_ZERO(&cpuset); + CPU_SET(SEQUENCER_READER_CORE, &cpuset); + pthread_attr_t attr_reader; + pthread_attr_init(&attr_reader); + pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); + reinterpret_cast(this)); } Sequencer::~Sequencer() { @@ -129,6 +128,8 @@ double PrefetchAll(Storage* storage, TxnProto* txn) { #endif void Sequencer::RunWriter() { + PrintCpu("RunWriter", 0); + Spin(1); #ifdef PAXOS @@ -165,71 +166,34 @@ void Sequencer::RunWriter() { string batch_string; batch.set_type(MessageProto::TXN_BATCH); - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; + for (int batch_number = configuration_->this_node_id; !deconstructor_invoked_; batch_number += configuration_->all_nodes.size()) { // Begin epoch. double epoch_start = GetTime(); batch.set_batch_number(batch_number); batch.clear_data(); -#ifdef PREFETCHING - // Include txn requests from earlier that have now had time to prefetch. - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - multimap::iterator it = fetching_txns.begin(); - if (it == fetching_txns.end() || it->first > GetTime() || - batch.data_size() >= MAX_BATCH_SIZE) { - break; - } - TxnProto* txn = it->second; - fetching_txns.erase(it); - string txn_string; - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } -#endif - // Collect txn requests for this epoch. int txn_id_offset = 0; while (!deconstructor_invoked_ && GetTime() < epoch_start + epoch_duration_) { // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { + if (batch.data_size() < MAX_LOCK_BATCH_SIZE) { TxnProto* txn; string txn_string; - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) { - sequencer_recv[txn->txn_id() / SAMPLE_RATE] = - epoch_start - + epoch_duration_ * (static_cast(rand()) / RAND_MAX); - } -#endif -#ifdef PREFETCHING - double wait_time = PrefetchAll(storage_, txn); - if (wait_time > 0) { - fetching_txns.insert(std::make_pair(epoch_start + wait_time, txn)); - } else { - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } -#else + client_->GetTxn(&txn, + batch_number * MAX_LOCK_BATCH_SIZE + txn_id_offset); + // Find a bad transaction - if(txn->txn_id() == -1) { + if (txn->txn_id() == -1) { delete txn; continue; } - txn->SerializeToString(&txn_string); batch.add_data(txn_string); txn_id_offset++; delete txn; -#endif } } @@ -248,6 +212,8 @@ void Sequencer::RunWriter() { } void Sequencer::RunReader() { + PrintCpu("RunReader", 0); + Spin(1); #ifdef PAXOS Paxos paxos(ZOOKEEPER_CONF, true); @@ -296,6 +262,17 @@ void Sequencer::RunReader() { TxnProto txn; txn.ParseFromString(batch_message.data(i)); + // -------- [ここから追加] -------- + // Read-Onlyトランザクションかどうかの判定 + if (txn.write_set_size() == 0 && txn.read_write_set_size() == 0) { + // 書き込み操作が一切ない場合、read_onlyフラグをtrueにする + txn.set_read_only(true); + } else { + // 1つでも書き込み操作があればfalse + txn.set_read_only(false); + } + // -------- [ここまで追加] -------- + #ifdef LATENCY_TEST if (txn.txn_id() % SAMPLE_RATE == 0) watched_txn = txn.txn_id(); @@ -352,8 +329,9 @@ void Sequencer::RunReader() { // Report output. if (GetTime() > time + 1) { #ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; + std::cout << "Submitted " << txn_count << " txns in " << batch_count + << " batches,\n" + << std::flush; #endif // Reset txn count. time = GetTime(); @@ -362,4 +340,4 @@ void Sequencer::RunReader() { } } Spin(1); -} +} \ No newline at end of file diff --git a/src_calvin_vector_vll/sequencer/sequencer.h b/src_calvin_ext/sequencer/sequencer.h old mode 100755 new mode 100644 similarity index 90% rename from src_calvin_vector_vll/sequencer/sequencer.h rename to src_calvin_ext/sequencer/sequencer.h index 50fc4f26..e638f332 --- a/src_calvin_vector_vll/sequencer/sequencer.h +++ b/src_calvin_ext/sequencer/sequencer.h @@ -11,20 +11,19 @@ #include #include -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 +#include "common/definitions.hh" -#define MAX_BATCH_SIZE 21 +// #define PAXOS +// #define PREFETCHING #define SAMPLES 100000 #define SAMPLE_RATE 999 -//#define LATENCY_TEST +// #define LATENCY_TEST +using std::queue; using std::set; using std::string; -using std::queue; class Configuration; class Connection; @@ -53,7 +52,9 @@ class Sequencer { public: // The constructor creates background threads and starts the Sequencer's main // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client, + Sequencer(Configuration* conf, + Connection* connection, + Client* client, Storage* storage); // Halts the main loops. @@ -77,8 +78,8 @@ class Sequencer { // Functions to start the Multiplexor's main loops, called in new pthreads by // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); + static void* RunSequencerWriter(void* arg); + static void* RunSequencerReader(void* arg); // Sets '*nodes' to contain the node_id of every node participating in 'txn'. void FindParticipatingNodes(const TxnProto& txn, set* nodes); diff --git a/src_calvin_4_partitions/tests/collapsed_versioned_storage_test.cc b/src_calvin_ext/tests/collapsed_versioned_storage_test.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_4_partitions/tests/collapsed_versioned_storage_test.cc rename to src_calvin_ext/tests/collapsed_versioned_storage_test.cc index 285b9134..dc3f1bec --- a/src_calvin_4_partitions/tests/collapsed_versioned_storage_test.cc +++ b/src_calvin_ext/tests/collapsed_versioned_storage_test.cc @@ -76,5 +76,3 @@ int main(int argc, char** argv) { CollapsedVersionedStorageTest(); CheckpointingTest(); } - - diff --git a/src_calvin_vector_vll/tests/configuration_test.cc b/src_calvin_ext/tests/configuration_test.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_vector_vll/tests/configuration_test.cc rename to src_calvin_ext/tests/configuration_test.cc index 79a569a5..417605a3 --- a/src_calvin_vector_vll/tests/configuration_test.cc +++ b/src_calvin_ext/tests/configuration_test.cc @@ -30,4 +30,3 @@ int main(int argc, char** argv) { ConfigurationTest_ReadFromFile(); ConfigurationTest_LookupPartition(); } - diff --git a/src_calvin_3_partitions/tests/connection_test.cc b/src_calvin_ext/tests/connection_test.cc old mode 100755 new mode 100644 similarity index 97% rename from src_calvin_3_partitions/tests/connection_test.cc rename to src_calvin_ext/tests/connection_test.cc index 8bf28f97..aba994ab --- a/src_calvin_3_partitions/tests/connection_test.cc +++ b/src_calvin_ext/tests/connection_test.cc @@ -158,9 +158,8 @@ TEST(LinkUnlinkChannelTest) { } */ int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); + // InprocTest(); + // RemoteTest(); + // ChannelNotCreatedYetTest(); + // LinkUnlinkChannelTest(); } - diff --git a/src_calvin_4_partitions/tests/deterministic_lock_manager_test.cc b/src_calvin_ext/tests/deterministic_lock_manager_test.cc old mode 100755 new mode 100644 similarity index 94% rename from src_calvin_4_partitions/tests/deterministic_lock_manager_test.cc rename to src_calvin_ext/tests/deterministic_lock_manager_test.cc index be34c6f8..376344d3 --- a/src_calvin_4_partitions/tests/deterministic_lock_manager_test.cc +++ b/src_calvin_ext/tests/deterministic_lock_manager_test.cc @@ -114,9 +114,9 @@ TEST(ThroughputTest) { args.SerializeToString(&args_string); for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); + // txns.push_back(new TxnProto()); + // for (int j = 0; j < 10; j++) + // txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); } @@ -140,8 +140,7 @@ TEST(ThroughputTest) { } int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); + // SimpleLockingTest(); + // LocksReleasedOutOfOrder(); ThroughputTest(); } - diff --git a/src_calvin_4_partitions/tests/deterministic_scheduler_test.cc b/src_calvin_ext/tests/deterministic_scheduler_test.cc old mode 100755 new mode 100644 similarity index 80% rename from src_calvin_4_partitions/tests/deterministic_scheduler_test.cc rename to src_calvin_ext/tests/deterministic_scheduler_test.cc index c0139888..2acd2366 --- a/src_calvin_4_partitions/tests/deterministic_scheduler_test.cc +++ b/src_calvin_ext/tests/deterministic_scheduler_test.cc @@ -6,5 +6,4 @@ #include "common/types.h" -int main(int argc, char** argv) { -} +int main(int argc, char** argv) {} diff --git a/src_calvin_3_partitions/tests/fetching_storage_test.cc b/src_calvin_ext/tests/fetching_storage_test.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_3_partitions/tests/fetching_storage_test.cc rename to src_calvin_ext/tests/fetching_storage_test.cc index 79a3e4ec..ed63cfed --- a/src_calvin_3_partitions/tests/fetching_storage_test.cc +++ b/src_calvin_ext/tests/fetching_storage_test.cc @@ -7,7 +7,6 @@ #include "common/testing.h" - TEST(FetchingStorageTest) { system("rm ../db/storage/*"); FetchingStorage* storage = FetchingStorage::BuildStorage(); @@ -32,5 +31,3 @@ TEST(FetchingStorageTest) { int main(int argc, char** argv) { FetchingStorageTest(); } - - diff --git a/src_calvin_4_partitions/tests/microbenchmark_test.cc b/src_calvin_ext/tests/microbenchmark_test.cc old mode 100755 new mode 100644 similarity index 71% rename from src_calvin_4_partitions/tests/microbenchmark_test.cc rename to src_calvin_ext/tests/microbenchmark_test.cc index 14b5b8fd..b0059250 --- a/src_calvin_4_partitions/tests/microbenchmark_test.cc +++ b/src_calvin_ext/tests/microbenchmark_test.cc @@ -17,11 +17,12 @@ using std::set; SimpleStorage* actual_storage; Configuration* config; -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) +#define CHECK_OBJECT(KEY, EXPECTED_VALUE) \ + do { \ + Value* actual_value; \ + actual_value = actual_storage->ReadObject(KEY); \ + EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ + } while (0) TEST(MicrobenchmarkTest) { config = new Configuration(0, "common/configuration_test_one_node.conf"); @@ -38,23 +39,21 @@ TEST(MicrobenchmarkTest) { txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); + StorageManager* storage = + new StorageManager(config, connection, actual_storage, txn); microbenchmark.Execute(txn, storage); // Check post-execution storage state. set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) + for (int i = 0; i < Microbenchmark::RW_SET_SIZE; i++) write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { + for (int i = 0; i < microbenchmark.DB_SIZE; i++) { if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); + CHECK_OBJECT(IntToString(i), IntToString(i + 1)); else CHECK_OBJECT(IntToString(i), IntToString(i)); } - - delete storage; delete txn; @@ -69,4 +68,3 @@ TEST(MicrobenchmarkTest) { int main(int argc, char** argv) { MicrobenchmarkTest(); } - diff --git a/src_calvin_3_partitions/tests/paxos_test.cc b/src_calvin_ext/tests/paxos_test.cc old mode 100755 new mode 100644 similarity index 97% rename from src_calvin_3_partitions/tests/paxos_test.cc rename to src_calvin_ext/tests/paxos_test.cc index 20bdfe09..c57e8dd8 --- a/src_calvin_3_partitions/tests/paxos_test.cc +++ b/src_calvin_ext/tests/paxos_test.cc @@ -18,7 +18,7 @@ int throughput; // Create Paxos object to submit some batches, inserting them into // a globally consistent batch order. -void* Writer(void *arg) { +void* Writer(void* arg) { // Create paxos object. Paxos writer(ZOOKEEPER_CONF, false); @@ -36,7 +36,7 @@ void* Writer(void *arg) { write_number++; // The interal is 0 to 1 msec. srand(50); - usleep((rand()%10)*100); + usleep((rand() % 10) * 100); // Test for 70 seconds. if (end - start > 70) break; @@ -46,7 +46,7 @@ void* Writer(void *arg) { } // Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { +void* Reader(void* arg) { // Create Paxos object. Paxos reader(ZOOKEEPER_CONF, true); diff --git a/src_calvin_vector_vll/tests/profile_test.cc b/src_calvin_ext/tests/profile_test.cc old mode 100755 new mode 100644 similarity index 93% rename from src_calvin_vector_vll/tests/profile_test.cc rename to src_calvin_ext/tests/profile_test.cc index f55e19f0..57f553fc --- a/src_calvin_vector_vll/tests/profile_test.cc +++ b/src_calvin_ext/tests/profile_test.cc @@ -8,7 +8,7 @@ int main(int argc, char** argv) { Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); + new Configuration(0, "common/configuration_test_one_node.conf"); CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); TPCC* tpcc = new TPCC(); diff --git a/src_calvin_vector_vll/tests/sequencer_test.cc b/src_calvin_ext/tests/sequencer_test.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_vector_vll/tests/sequencer_test.cc rename to src_calvin_ext/tests/sequencer_test.cc index 12c5b6b9..1fe5eb52 --- a/src_calvin_vector_vll/tests/sequencer_test.cc +++ b/src_calvin_ext/tests/sequencer_test.cc @@ -13,4 +13,3 @@ TEST(SequencerTest) { int main(int argc, char** argv) { SequencerTest(); } - diff --git a/src_calvin_3_partitions/tests/serial_scheduler_test.cc b/src_calvin_ext/tests/serial_scheduler_test.cc old mode 100755 new mode 100644 similarity index 100% rename from src_calvin_3_partitions/tests/serial_scheduler_test.cc rename to src_calvin_ext/tests/serial_scheduler_test.cc diff --git a/src_calvin_4_partitions/tests/simple_storage_test.cc b/src_calvin_ext/tests/simple_storage_test.cc old mode 100755 new mode 100644 similarity index 99% rename from src_calvin_4_partitions/tests/simple_storage_test.cc rename to src_calvin_ext/tests/simple_storage_test.cc index 6a855366..c0610cc8 --- a/src_calvin_4_partitions/tests/simple_storage_test.cc +++ b/src_calvin_ext/tests/simple_storage_test.cc @@ -23,5 +23,3 @@ TEST(SimpleStorageTest) { int main(int argc, char** argv) { SimpleStorageTest(); } - - diff --git a/src_calvin_vector_vll/tests/storage_manager_test.cc b/src_calvin_ext/tests/storage_manager_test.cc old mode 100755 new mode 100644 similarity index 97% rename from src_calvin_vector_vll/tests/storage_manager_test.cc rename to src_calvin_ext/tests/storage_manager_test.cc index be719e5c..1adee300 --- a/src_calvin_vector_vll/tests/storage_manager_test.cc +++ b/src_calvin_ext/tests/storage_manager_test.cc @@ -135,8 +135,7 @@ TEST(TwoNodes) { } int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); + // TODO(alex): Fix these tests! + // SingleNode(); + // TwoNodes(); } - diff --git a/src_calvin_vector_vll/tests/tpcc_test.cc b/src_calvin_ext/tests/tpcc_test.cc old mode 100755 new mode 100644 similarity index 90% rename from src_calvin_vector_vll/tests/tpcc_test.cc rename to src_calvin_ext/tests/tpcc_test.cc index 8dff4592..568b481e --- a/src_calvin_vector_vll/tests/tpcc_test.cc +++ b/src_calvin_ext/tests/tpcc_test.cc @@ -12,7 +12,7 @@ // We make these global variables to avoid weird pointer passing and code // redundancy Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); + new Configuration(0, "common/configuration_test_one_node.conf"); ConnectionMultiplexer* multiplexer; Connection* connection; SimpleStorage* simple_store; @@ -22,10 +22,9 @@ TPCC* tpcc; TEST(IdGenerationTest) { EXPECT_EQ(tpcc->CheckpointID("w1"), 1); EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); + EXPECT_EQ(tpcc->CheckpointID("c1"), + WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + 1); + EXPECT_EQ(tpcc->CheckpointID("w2si1"), 1000000 + 2 * NUMBER_OF_ITEMS + 1); EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); @@ -217,8 +216,7 @@ TEST(InitializeTest) { for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { char district_key[128]; Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); + snprintf(district_key, sizeof(district_key), "w%dd%d", i, j); district_value = simple_store->ReadObject(district_key); District* dummy_district = new District(); @@ -229,8 +227,7 @@ TEST(InitializeTest) { for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { char customer_key[128]; Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); + snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", i, j, k); customer_value = simple_store->ReadObject(customer_key); Customer* dummy_customer = new Customer(); @@ -244,8 +241,7 @@ TEST(InitializeTest) { char item_key[128], stock_key[128]; Value* stock_value; snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); + snprintf(stock_key, sizeof(stock_key), "%ss%s", warehouse_key, item_key); stock_value = simple_store->ReadObject(stock_key); Stock* dummy_stock = new Stock(); @@ -256,14 +252,14 @@ TEST(InitializeTest) { // Expect all items to be there for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; + char item_key[128]; + Value item_value; + snprintf(item_key, sizeof(item_key), "i%d", i); + item_value = *(tpcc->GetItem(string(item_key))); + + Item* dummy_item = new Item(); + EXPECT_TRUE(dummy_item->ParseFromString(item_value)); + delete dummy_item; } END; @@ -292,11 +288,11 @@ TEST(NewOrderTest) { txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); + StorageManager* storage = + new StorageManager(config, connection, simple_store, txn); // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); + District* district = new District(); Value* district_value; district_value = storage->ReadObject(txn->read_write_set(0)); assert(district->ParseFromString(*district_value)); @@ -403,15 +399,15 @@ TEST(NewOrderTest) { // Check the order line // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); + // Value* order_line_value; + // OrderLine* order_line = new OrderLine(); + // order_line_value = storage->ReadObject(txn->write_set(i)); + // EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); + // EXPECT_EQ(order_line->amount(), item->price() * + // txn_args->quantities(i)); EXPECT_EQ(order_line->number(), i); // Free memory -// delete order_line; + // delete order_line; delete item; delete stock; } @@ -447,25 +443,25 @@ TEST(PaymentTest) { txn->add_read_set(txn->write_set(0)); txn->add_readers(0); txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); + StorageManager* storage = + new StorageManager(config, connection, simple_store, txn); // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); + Warehouse* warehouse = new Warehouse(); Value* warehouse_value; warehouse_value = storage->ReadObject(txn->read_write_set(0)); assert(warehouse->ParseFromString(*warehouse_value)); int old_warehouse_year_to_date = warehouse->year_to_date(); // Prefetch district - District *district = new District(); + District* district = new District(); Value* district_value; district_value = storage->ReadObject(txn->read_write_set(1)); assert(district->ParseFromString(*district_value)); int old_district_year_to_date = district->year_to_date(); // Preetch customer - Customer *customer = new Customer(); + Customer* customer = new Customer(); Value* customer_value; customer_value = storage->ReadObject(txn->read_write_set(2)); assert(customer->ParseFromString(*customer_value)); @@ -488,10 +484,10 @@ TEST(PaymentTest) { assert(customer->ParseFromString(*customer_value)); // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); + EXPECT_EQ(warehouse->year_to_date(), + old_warehouse_year_to_date + txn_args->amount()); + EXPECT_EQ(district->year_to_date(), + old_district_year_to_date + txn_args->amount()); EXPECT_EQ(customer->year_to_date_payment(), old_customer_year_to_date_payment + txn_args->amount()); EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); @@ -574,4 +570,3 @@ int main(int argc, char** argv) { return 0; } - diff --git a/src_calvin_vector_vll/tests/types_test.cc b/src_calvin_ext/tests/types_test.cc old mode 100755 new mode 100644 similarity index 95% rename from src_calvin_vector_vll/tests/types_test.cc rename to src_calvin_ext/tests/types_test.cc index ef5bc102..485b9d24 --- a/src_calvin_vector_vll/tests/types_test.cc +++ b/src_calvin_ext/tests/types_test.cc @@ -5,7 +5,7 @@ #include "common/testing.h" TEST(PackSignedIntTest) { - int8 i1 = 65; + int8 i1 = 65; int16 i2 = -2551; int32 i3 = 0; int64 i4 = -2551255125512551; @@ -19,7 +19,7 @@ TEST(PackSignedIntTest) { } TEST(PackUnsignedIntTest) { - uint8 u1 = 251; + uint8 u1 = 251; uint16 u2 = 2551; uint32 u3 = 0; uint64 u4 = 2551255125512551; diff --git a/src_calvin_vector_vll/Makefile b/src_calvin_vector_vll/Makefile deleted file mode 100755 index e3e6e17d..00000000 --- a/src_calvin_vector_vll/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_calvin_vector_vll/Makefile.template b/src_calvin_vector_vll/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_calvin_vector_vll/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_calvin_vector_vll/applications/Makefile.inc b/src_calvin_vector_vll/applications/Makefile.inc deleted file mode 100644 index db42fd1b..00000000 --- a/src_calvin_vector_vll/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/applications/tpcc.cc b/src_calvin_vector_vll/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_calvin_vector_vll/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_calvin_vector_vll/applications/tpcc.h b/src_calvin_vector_vll/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_calvin_vector_vll/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_calvin_vector_vll/backend/Makefile.inc b/src_calvin_vector_vll/backend/Makefile.inc deleted file mode 100644 index edcf67bd..00000000 --- a/src_calvin_vector_vll/backend/Makefile.inc +++ /dev/null @@ -1,28 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/checkpointable_storage.cc \ - backend/collapsed_versioned_storage.cc \ - backend/fetching_storage.cc \ - backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/backend/collapsed_versioned_storage.h b/src_calvin_vector_vll/backend/collapsed_versioned_storage.h deleted file mode 100644 index 249a40d6..00000000 --- a/src_calvin_vector_vll/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_calvin_vector_vll/backend/fetching_storage.cc b/src_calvin_vector_vll/backend/fetching_storage.cc deleted file mode 100644 index f5514372..00000000 --- a/src_calvin_vector_vll/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_calvin_vector_vll/backend/fetching_storage.h b/src_calvin_vector_vll/backend/fetching_storage.h deleted file mode 100644 index 273dc04c..00000000 --- a/src_calvin_vector_vll/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_calvin_vector_vll/backend/simple_storage.cc b/src_calvin_vector_vll/backend/simple_storage.cc deleted file mode 100644 index 46237c8f..00000000 --- a/src_calvin_vector_vll/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_calvin_vector_vll/backend/simple_storage.h b/src_calvin_vector_vll/backend/simple_storage.h deleted file mode 100644 index f15f69e9..00000000 --- a/src_calvin_vector_vll/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_calvin_vector_vll/backend/storage.h b/src_calvin_vector_vll/backend/storage.h deleted file mode 100644 index fa2e4efb..00000000 --- a/src_calvin_vector_vll/backend/storage.h +++ /dev/null @@ -1,49 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_calvin_vector_vll/backend/storage_manager.cc b/src_calvin_vector_vll/backend/storage_manager.cc deleted file mode 100644 index 7de4ff40..00000000 --- a/src_calvin_vector_vll/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == configuration_->this_node_id) - reader = true; - } - - if (reader) { - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != configuration_->this_node_id) { - message.set_destination_node(txn->writers(i)); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == configuration_->this_node_id) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_calvin_vector_vll/backend/storage_manager.h b/src_calvin_vector_vll/backend/storage_manager.h deleted file mode 100644 index a374017a..00000000 --- a/src_calvin_vector_vll/backend/storage_manager.h +++ /dev/null @@ -1,87 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_calvin_vector_vll/backend/versioned_storage.h b/src_calvin_vector_vll/backend/versioned_storage.h deleted file mode 100644 index 24c6223b..00000000 --- a/src_calvin_vector_vll/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_calvin_vector_vll/common/Makefile.inc b/src_calvin_vector_vll/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_calvin_vector_vll/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/common/configuration.cc b/src_calvin_vector_vll/common/configuration.cc deleted file mode 100755 index c0dd1d2e..00000000 --- a/src_calvin_vector_vll/common/configuration.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_calvin_vector_vll/common/configuration.h b/src_calvin_vector_vll/common/configuration.h deleted file mode 100755 index d7ce7fb5..00000000 --- a/src_calvin_vector_vll/common/configuration.h +++ /dev/null @@ -1,95 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/types.h" - - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - -struct Node { - // Globally unique node identifier. - int node_id; - int replica_id; - int partition_id; - - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_calvin_vector_vll/common/configuration_test.conf b/src_calvin_vector_vll/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_calvin_vector_vll/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_calvin_vector_vll/common/configuration_test_one_node.conf b/src_calvin_vector_vll/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_calvin_vector_vll/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_calvin_vector_vll/common/connection.h b/src_calvin_vector_vll/common/connection.h deleted file mode 100755 index 2965a450..00000000 --- a/src_calvin_vector_vll/common/connection.h +++ /dev/null @@ -1,197 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_calvin_vector_vll/common/test_template.cc b/src_calvin_vector_vll/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_calvin_vector_vll/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_calvin_vector_vll/common/testing.h b/src_calvin_vector_vll/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_calvin_vector_vll/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_calvin_vector_vll/common/types.h b/src_calvin_vector_vll/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_calvin_vector_vll/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_calvin_vector_vll/common/utils.h b/src_calvin_vector_vll/common/utils.h deleted file mode 100755 index ea1021cb..00000000 --- a/src_calvin_vector_vll/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_calvin_vector_vll/common/zmq.hpp b/src_calvin_vector_vll/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_calvin_vector_vll/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_calvin_vector_vll/deployment/Makefile.inc b/src_calvin_vector_vll/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_calvin_vector_vll/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_calvin_vector_vll/deployment/addhosts.pl b/src_calvin_vector_vll/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_calvin_vector_vll/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_calvin_vector_vll/deployment/main.cc b/src_calvin_vector_vll/deployment/main.cc deleted file mode 100755 index 4e95f433..00000000 --- a/src_calvin_vector_vll/deployment/main.cc +++ /dev/null @@ -1,180 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 10000 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other; - do { - other = rand() % config_->all_nodes.size(); - } while (other == config_->this_node_id); - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other); - } else { - // Single-partition txn. - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, - storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_calvin_vector_vll/deployment/makeconf.pl b/src_calvin_vector_vll/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_calvin_vector_vll/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_calvin_vector_vll/deployment/portfile b/src_calvin_vector_vll/deployment/portfile deleted file mode 100755 index fae03c52..00000000 --- a/src_calvin_vector_vll/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -52984 diff --git a/src_calvin_vector_vll/deployment/run.pbs b/src_calvin_vector_vll/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_calvin_vector_vll/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_calvin_vector_vll/deployment/test.conf b/src_calvin_vector_vll/deployment/test.conf deleted file mode 100755 index 36bc2882..00000000 --- a/src_calvin_vector_vll/deployment/test.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:1:12:128.36.232.12:51261 #rhino -node2=0:2:12:128.36.232.15:51262 # gator -node3=0:3:12:128.36.232.20:51261 #macaw -node4=0:4:12:128.36.232.19:51262 # peacock -node5=0:5:12:128.36.232.34:51261 # perch -node6=0:6:12:128.36.232.14:51262 # jaguar -node7=0:7:12:128.36.232.16:51261 # hippo -node8=0:8:12:128.36.232.39:51261 #viper -node9=0:9:12:128.36.232.7:51261 #hornet -node10=0:10:12:128.36.232.8:51261 #cricket -node11=0:11:12:128.36.232.28:51261 #termite -node12=0:12:12:128.36.232.5:51261 #cicada -node13=0:13:12:128.36.232.6:51261 #bumblebee -node14=0:14:12:128.36.232.44:51261 #frog -node15=0:15:12:128.36.232.50:51261 #tick -node16=0:16:12:128.36.232.42:51261 #turtle -node17=0:17:12:128.36.232.35:51261 #aphid -node18=0:18:12:128.36.232.41:51261 #chameleon -node19=0:19:12:128.36.232.38:51261 #rattlesnake -node20=0:20:12:128.36.232.45:51261 #scorpion -node21=0:21:12:128.36.232.29:51261 #cardinal -node22=0:22:12:128.36.232.24:51261 #swan -node23=0:23:12:128.36.232.27:51261 #monkey - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_calvin_vector_vll/obj/.deps b/src_calvin_vector_vll/obj/.deps deleted file mode 100644 index e69de29b..00000000 diff --git a/src_calvin_vector_vll/obj/common/configuration.d b/src_calvin_vector_vll/obj/common/configuration.d deleted file mode 100644 index d4246650..00000000 --- a/src_calvin_vector_vll/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_calvin_vector_vll/paxos/Makefile.inc b/src_calvin_vector_vll/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_calvin_vector_vll/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/paxos/zookeeper.conf b/src_calvin_vector_vll/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_calvin_vector_vll/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_calvin_vector_vll/proto/Makefile.inc b/src_calvin_vector_vll/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_calvin_vector_vll/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_calvin_vector_vll/proto/tpcc.proto b/src_calvin_vector_vll/proto/tpcc.proto deleted file mode 100755 index 50088661..00000000 --- a/src_calvin_vector_vll/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_calvin_vector_vll/proto/tpcc_args.proto b/src_calvin_vector_vll/proto/tpcc_args.proto deleted file mode 100755 index 248051c8..00000000 --- a/src_calvin_vector_vll/proto/tpcc_args.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - - optional int32 order_number = 41; - - optional int32 lastest_order_number = 42; - optional int32 threshold = 51; -} diff --git a/src_calvin_vector_vll/proto/txn.proto b/src_calvin_vector_vll/proto/txn.proto deleted file mode 100755 index 2693e4ae..00000000 --- a/src_calvin_vector_vll/proto/txn.proto +++ /dev/null @@ -1,58 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - BLOCKED = 4; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; -} - diff --git a/src_calvin_vector_vll/scheduler/Makefile.inc b/src_calvin_vector_vll/scheduler/Makefile.inc deleted file mode 100755 index f5678dcc..00000000 --- a/src_calvin_vector_vll/scheduler/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc \ - scheduler/serial_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/scheduler/deterministic_lock_manager.h b/src_calvin_vector_vll/scheduler/deterministic_lock_manager.h deleted file mode 100755 index c84dad7b..00000000 --- a/src_calvin_vector_vll/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 500000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_calvin_vector_vll/scheduler/deterministic_scheduler.cc b/src_calvin_vector_vll/scheduler/deterministic_scheduler.cc deleted file mode 100755 index 2295796b..00000000 --- a/src_calvin_vector_vll/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,410 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/zmq.hpp" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" -#include "applications/tpcc.h" -#include "common/types.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using zmq::socket_t; -using std::bitset; - -static void DeleteTxnPtr(void* data, void* hint) { free(data); } - -void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { - TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); - *txn_ptr = txn; - zmq::message_t msg(txn_ptr, sizeof(*txn_ptr), DeleteTxnPtr, NULL); - socket->send(msg); -} - -TxnProto* DeterministicScheduler::GetTxnPtr(socket_t* socket, - zmq::message_t* msg) { - if (!socket->recv(msg, ZMQ_NOBLOCK)) - return NULL; - TxnProto* txn = *reinterpret_cast(msg->data()); - return txn; -} - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - Connection* batch_connection, - Storage* storage, - const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - pthread_mutex_init(&mutex_, NULL); - - txns_queue = new AtomicQueue(); - done_queue = new AtomicQueue(); - - for (int i = 0; i < NUM_THREADS; i++) { - message_queues[i] = new AtomicQueue(); - } - - -Spin(2); - - // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(7, &cpuset); - pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); - pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, - reinterpret_cast(this)); - - -// pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { - string channel("scheduler"); - channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); - - pthread_attr_t attr; - pthread_attr_init(&attr); - CPU_ZERO(&cpuset); - if (i == 0 || i == 1) - CPU_SET(i, &cpuset); - else - CPU_SET(i+2, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, - reinterpret_cast( - new pair(i, this))); - } -} - -void UnfetchAll(Storage* storage, TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (StringToInt(txn->read_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_set(i)); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (StringToInt(txn->read_write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_write_set(i)); - for (int i = 0; i < txn->write_set_size(); i++) - if (StringToInt(txn->write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->write_set(i)); -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - int thread = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - unordered_map active_txns; - - // Begin main loop. - MessageProto message; - while (true) { - bool got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - // Remote read result. - assert(message.type() == MessageProto::READ_RESULT); - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - // Execute and clean up. - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - delete manager; - - scheduler->thread_connections_[thread]-> - UnlinkChannel(IntToString(txn->txn_id())); - active_txns.erase(message.destination_channel()); - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } - } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { - // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } else { - scheduler->thread_connections_[thread]-> - LinkChannel(IntToString(txn->txn_id())); - // There are outstanding remote reads. - active_txns[IntToString(txn->txn_id())] = manager; - } - } - } - } - return NULL; -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % 1000000; -} - -// Returns ptr to heap-allocated -unordered_map batches; -MessageProto* GetBatch(int batch_id, Connection* connection) { - if (batches.count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[batch_id]; - batches.erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -void* DeterministicScheduler::LockManagerThread(void* arg) { - - DeterministicScheduler* scheduler = reinterpret_cast(arg); - - map TxnsQueue; - vector Cx(1000000, 0); - vector Cs(1000000, 0); - - // Run main loop. - MessageProto message; - MessageProto* batch_message = NULL; - int txns = 0; - double time = GetTime(); - int blocked_txns = 0; - int batch_offset = 0; - int batch_number = 0; - - int sca = 0; - - int hash_index; - bitset<1000000> Dx; - bitset<1000000> Ds; - - Configuration* configuration = scheduler->configuration_; - int this_node_id = configuration->this_node_id; - - while (true) { - TxnProto* done_txn; - bool got_it = scheduler->done_queue->Pop(&done_txn); - if (got_it == true) { - // We have received a finished transaction back, release the locks - for (int i = 0; i < done_txn->read_write_set_size(); i++) { - if (configuration->LookupPartition(done_txn->read_write_set(i)) == this_node_id) { - hash_index = Hash(done_txn->read_write_set(i)); - Cx[hash_index]--; - } - } - for (int i = 0; i < done_txn->read_set_size(); i++) { - if (configuration->LookupPartition(done_txn->read_set(i)) == this_node_id) { - hash_index = Hash(done_txn->read_set(i)); - Cs[hash_index]--; - } - } - - // Remove the transaction from TxnsQueue; - TxnsQueue.erase(done_txn->txn_id()); - - //lock_manager_->Release(done_txn); - - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) - txns++; - - delete done_txn; - - // If the first action in the ActionQueue is BLOCKED, execute it. - if (!TxnsQueue.empty()) { - TxnProto* txn = TxnsQueue.begin()->second; - if (txn->status() == TxnProto::BLOCKED) { - blocked_txns--; - txn->set_status(TxnProto::ACTIVE); - scheduler->txns_queue->Push(txn); - } - } - - } else { - // Have we run out of txns in our batch? Let's get some new ones. - if (batch_message == NULL) { - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Done with current batch, get next. - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Current batch has remaining txns, grab up to 10.&& TxnsQueue.size() < 400 - } else if (blocked_txns < 20) { - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - - // Request write locks. - for (int i = 0; i < txn->read_write_set_size(); i++) { - if (configuration->LookupPartition(txn->read_write_set(i)) == this_node_id) { - hash_index = Hash(txn->read_write_set(i)); - Cx[hash_index]++; - if (Cx[hash_index] > 1 || Cs[hash_index] > 0) { - txn->set_status(TxnProto::BLOCKED); - } - } - } - - // Request read locks. - for (int i = 0; i < txn->read_set_size(); i++) { - if (configuration->LookupPartition(txn->read_set(i)) == this_node_id) { - hash_index = Hash(txn->read_set(i)); - Cs[hash_index]++; - if (Cx[hash_index] > 0) { - txn->set_status(TxnProto::BLOCKED); - } - } - } - - TxnsQueue.insert(std::pair(txn->txn_id(), txn)); - if (txn->status() == TxnProto::ACTIVE) { - scheduler->txns_queue->Push(txn); - } else { - blocked_txns ++; - } - } else { -sca++; - // Create our 100KB bit arrays - Dx.reset(); - Ds.reset(); - - for (std::map::iterator it = TxnsQueue.begin(); - it != TxnsQueue.end(); ++it) { - TxnProto* txn = it->second; - - // Check whether the Blocked actions can safely be run - if (txn->status() == TxnProto::BLOCKED) { - bool success = true; - - // Check for conflicts in WriteSet - for (int i = 0; i < txn->read_write_set_size(); i++) { - if (configuration->LookupPartition(txn->read_write_set(i)) == this_node_id) { - hash_index = Hash(txn->read_write_set(i)); - if (Dx[hash_index] == 1 || Ds[hash_index] == 1) { - success = false; - } - Dx[hash_index] = 1; - } - } - - // Check for conflicts in ReadSet - for (int i = 0; i < txn->read_set_size(); i++) { - if (configuration->LookupPartition(txn->read_set(i)) == this_node_id) { - hash_index = Hash(txn->read_set(i)); - if (Dx[hash_index] == 1) { - success = false; - } - Ds[hash_index] = 1; - } - } - - if (success == true) { - blocked_txns--; - txn->set_status(TxnProto::ACTIVE); - scheduler->txns_queue->Push(txn); - } - - } else { - // If the transaction is free, just mark the bit-array - for (int i = 0; i < txn->read_write_set_size(); i++) { - if (configuration->LookupPartition(txn->read_write_set(i)) == this_node_id) { - hash_index = Hash(txn->read_write_set(i)); - Dx[hash_index] = 1; - } - } - for (int i = 0; i < txn->read_set_size(); i++) { - if (configuration->LookupPartition(txn->read_set(i)) == this_node_id) { - hash_index = Hash(txn->read_set(i)); - Ds[hash_index] = 1; - } - } - } - } - } - } - - - // Report throughput. - if (GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout << "Completed " << (static_cast(txns) / total_time) - << " txns/sec, " << sca << " " - << (int)TxnsQueue.size() << " Queueing, " - << blocked_txns << " Blocking\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - sca = 0; - } - } - return NULL; -} diff --git a/src_calvin_vector_vll/scheduler/deterministic_scheduler.h b/src_calvin_vector_vll/scheduler/deterministic_scheduler.h deleted file mode 100755 index e78f4e45..00000000 --- a/src_calvin_vector_vll/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,95 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include -#include -#include - -#include "scheduler/scheduler.h" -#include "common/types.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -using std::deque; -using std::map; -using std::vector; - -namespace zmq { -class socket_t; -class message_t; -} -using zmq::socket_t; - -class Configuration; -class Connection; -class DeterministicLockManager; -class Storage; -class TxnProto; - -#define NUM_THREADS 4 -// #define PREFETCHING - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - static void* LockManagerThread(void* arg); - - void SendTxnPtr(socket_t* socket, TxnProto* txn); - TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - pthread_t lock_manager_thread_; - Connection* thread_connections_[NUM_THREADS]; - - // Connection for receiving txn batches from sequencer. - Connection* batch_connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Application currently being run. - const Application* application_; - - // The per-node lock manager tracks what transactions have temporary ownership - // of what database objects, allowing the scheduler to track LOCAL conflicts - // and enforce equivalence to transaction orders. - DeterministicLockManager* lock_manager_; - - // Sockets for communication between main scheduler thread and worker threads. - //socket_t* requests_out_; - //socket_t* requests_in_; - //socket_t* responses_out_[NUM_THREADS]; - //socket_t* responses_in_; - - pthread_mutex_t mutex_; - - AtomicQueue* txns_queue; - AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_calvin_vector_vll/scheduler/lock_manager.h b/src_calvin_vector_vll/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_calvin_vector_vll/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_calvin_vector_vll/scheduler/serial_scheduler.cc b/src_calvin_vector_vll/scheduler/serial_scheduler.cc deleted file mode 100755 index 3e532069..00000000 --- a/src_calvin_vector_vll/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_calvin_vector_vll/sequencer/Makefile.inc b/src_calvin_vector_vll/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_calvin_vector_vll/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_calvin_vector_vll/sequencer/sequencer.cc b/src_calvin_vector_vll/sequencer/sequencer.cc deleted file mode 100755 index 07995383..00000000 --- a/src_calvin_vector_vll/sequencer/sequencer.cc +++ /dev/null @@ -1,365 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), storage_(storage), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - // Start Sequencer main loops running in background thread. - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -#ifdef PREFETCHING -double PrefetchAll(Storage* storage, TxnProto* txn) { - double max_wait_time = 0; - double wait_time = 0; - for (int i = 0; i < txn->read_set_size(); i++) { - storage->Prefetch(txn->read_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - storage->Prefetch(txn->read_write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->write_set_size(); i++) { - storage->Prefetch(txn->write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) - prefetch_cold[txn->txn_id() / SAMPLE_RATE] = max_wait_time; -#endif - return max_wait_time; -} -#endif - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - -#ifdef PREFETCHING - multimap fetching_txns; -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - std::cout << "Starting sequencer.\n" << std::flush; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); - - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - -#ifdef PREFETCHING - // Include txn requests from earlier that have now had time to prefetch. - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - multimap::iterator it = fetching_txns.begin(); - if (it == fetching_txns.end() || it->first > GetTime() || - batch.data_size() >= MAX_BATCH_SIZE) { - break; - } - TxnProto* txn = it->second; - fetching_txns.erase(it); - string txn_string; - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } -#endif - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) { - sequencer_recv[txn->txn_id() / SAMPLE_RATE] = - epoch_start - + epoch_duration_ * (static_cast(rand()) / RAND_MAX); - } -#endif -#ifdef PREFETCHING - double wait_time = PrefetchAll(storage_, txn); - if (wait_time > 0) { - fetching_txns.insert(std::make_pair(epoch_start + wait_time, txn)); - } else { - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } -#else - if(txn->txn_id() == -1) { -//printf("Find a bad transaction,txn_id is %d. \n",(int)txn->txn_id()); - delete txn; - continue; - } - - - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; -#endif - } - } -//printf("Batch size is %d\n", batch.data_size()); - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - - // Set up batch messages for each system node. - map batches; - for (map::iterator it = configuration_->all_nodes.begin(); - it != configuration_->all_nodes.end(); ++it) { - batches[it->first].set_destination_channel("scheduler_"); - batches[it->first].set_destination_node(it->first); - batches[it->first].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif - - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - } while (!got_batch); -#endif - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - -#ifdef LATENCY_TEST - if (txn.txn_id() % SAMPLE_RATE == 0) - watched_txn = txn.txn_id(); -#endif - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - - bytes txn_data; - txn.SerializeToString(&txn_data); - - // Compute union of 'readers' and 'writers' (store in 'readers'). - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - // Insert txn into appropriate batches. - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - batches[*it].add_data(txn_data); - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - -#ifdef LATENCY_TEST - if (watched_txn != -1) { - sequencer_send[watched_txn] = GetTime(); - watched_txn = -1; - } -#endif - - // Report output. - if (GetTime() > time + 1) { -#ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; -#endif - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_calvin_vector_vll/tests/collapsed_versioned_storage_test.cc b/src_calvin_vector_vll/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_calvin_vector_vll/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_calvin_vector_vll/tests/connection_test.cc b/src_calvin_vector_vll/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_calvin_vector_vll/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_calvin_vector_vll/tests/deterministic_lock_manager_test.cc b/src_calvin_vector_vll/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_calvin_vector_vll/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_calvin_vector_vll/tests/deterministic_scheduler_test.cc b/src_calvin_vector_vll/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_calvin_vector_vll/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_calvin_vector_vll/tests/fetching_storage_test.cc b/src_calvin_vector_vll/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_calvin_vector_vll/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_calvin_vector_vll/tests/microbenchmark_test.cc b/src_calvin_vector_vll/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_calvin_vector_vll/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_calvin_vector_vll/tests/paxos_test.cc b/src_calvin_vector_vll/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_calvin_vector_vll/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_calvin_vector_vll/tests/serial_scheduler_test.cc b/src_calvin_vector_vll/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_calvin_vector_vll/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_calvin_vector_vll/tests/simple_storage_test.cc b/src_calvin_vector_vll/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_calvin_vector_vll/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_dependent_remote_index/applications/application.h b/src_dependent_remote_index/applications/application.h deleted file mode 100755 index 047e8513..00000000 --- a/src_dependent_remote_index/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, - Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_dependent_remote_index/applications/microbenchmark.cc b/src_dependent_remote_index/applications/microbenchmark.cc deleted file mode 100755 index e5efaa2b..00000000 --- a/src_dependent_remote_index/applications/microbenchmark.cc +++ /dev/null @@ -1,195 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnDependentSP(int64 txn_id, int64 index_number) { - TxnProto* txn = new TxnProto(); - txn->set_txn_id(txn_id); - txn->set_index_number(index_number); - txn->set_txn_type(MICROTXN_DEPENDENT_SP); - return txn; -} - -// Create a dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnDependentMP(int64 txn_id, int part1, int part2, int64 index_number) { - TxnProto* txn = new TxnProto(); - txn->set_txn_id(txn_id); - txn->set_index_number(index_number); - txn->set_index_node(part1); - txn->set_other_node(part2); - txn->set_txn_type(MICROTXN_DEPENDENT_MP); - - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey2)); - - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// Create a non-dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { - assert(part1 != part2 || nparts == 1); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - *val = IntToString(StringToInt(*val) + 1); - // Not necessary since storage already has a pointer to val. - // storage->PutObject(txn->read_write_set(i), val); - // Uncomment the following code if tested "short" transaction - int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - } - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, - Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { -#ifdef PREFETCHING - if (i % 10000 == 0) - std::cout << i << std::endl; - storage->Prefetch(IntToString(i), &wait_time); - storage->PutObject(IntToString(i), new Value(IntToString(i))); - if (i > COLD_CUTOFF) { - storage->Unfetch(IntToString(i)); - if (i % 10 == 0) - std::cout << i << std::endl; - } -#else - storage->PutObject(IntToString(i), new Value(IntToString(i))); -#endif - } - } -} - diff --git a/src_dependent_remote_index/applications/microbenchmark.h b/src_dependent_remote_index/applications/microbenchmark.h deleted file mode 100755 index 9e0afc72..00000000 --- a/src_dependent_remote_index/applications/microbenchmark.h +++ /dev/null @@ -1,59 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - MICROTXN_DEPENDENT_SP = 3, - MICROTXN_DEPENDENT_MP = 4, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2); - TxnProto* MicroTxnDependentSP(int64 txn_id, int64 index_number); - TxnProto* MicroTxnDependentMP(int64 txn_id, int part1, int part2, int64 index_number); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_dependent_remote_index/applications/tpcc.cc b/src_dependent_remote_index/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_dependent_remote_index/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_dependent_remote_index/applications/tpcc.h b/src_dependent_remote_index/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_dependent_remote_index/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_dependent_remote_index/backend/Makefile.inc b/src_dependent_remote_index/backend/Makefile.inc deleted file mode 100644 index edcf67bd..00000000 --- a/src_dependent_remote_index/backend/Makefile.inc +++ /dev/null @@ -1,28 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/checkpointable_storage.cc \ - backend/collapsed_versioned_storage.cc \ - backend/fetching_storage.cc \ - backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_remote_index/backend/collapsed_versioned_storage.cc b/src_dependent_remote_index/backend/collapsed_versioned_storage.cc deleted file mode 100644 index aa0f99f9..00000000 --- a/src_dependent_remote_index/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_dependent_remote_index/backend/collapsed_versioned_storage.h b/src_dependent_remote_index/backend/collapsed_versioned_storage.h deleted file mode 100644 index 249a40d6..00000000 --- a/src_dependent_remote_index/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_dependent_remote_index/backend/fetching_storage.cc b/src_dependent_remote_index/backend/fetching_storage.cc deleted file mode 100644 index f5514372..00000000 --- a/src_dependent_remote_index/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_dependent_remote_index/backend/fetching_storage.h b/src_dependent_remote_index/backend/fetching_storage.h deleted file mode 100644 index 273dc04c..00000000 --- a/src_dependent_remote_index/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_dependent_remote_index/backend/simple_storage.cc b/src_dependent_remote_index/backend/simple_storage.cc deleted file mode 100644 index 46237c8f..00000000 --- a/src_dependent_remote_index/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_dependent_remote_index/backend/simple_storage.h b/src_dependent_remote_index/backend/simple_storage.h deleted file mode 100644 index f15f69e9..00000000 --- a/src_dependent_remote_index/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_dependent_remote_index/backend/versioned_storage.h b/src_dependent_remote_index/backend/versioned_storage.h deleted file mode 100644 index 24c6223b..00000000 --- a/src_dependent_remote_index/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_dependent_remote_index/common/Makefile.inc b/src_dependent_remote_index/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_dependent_remote_index/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_remote_index/common/configuration.cc b/src_dependent_remote_index/common/configuration.cc deleted file mode 100755 index b07312ee..00000000 --- a/src_dependent_remote_index/common/configuration.cc +++ /dev/null @@ -1,119 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_dependent_remote_index/common/configuration.h b/src_dependent_remote_index/common/configuration.h deleted file mode 100755 index 43a1289e..00000000 --- a/src_dependent_remote_index/common/configuration.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/types.h" - - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - -struct Node { - // Globally unique node identifier. - int node_id; - int replica_id; - int partition_id; - - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_dependent_remote_index/common/configuration_test.conf b/src_dependent_remote_index/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_dependent_remote_index/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_dependent_remote_index/common/connection.cc b/src_dependent_remote_index/common/connection.cc deleted file mode 100755 index 8466c191..00000000 --- a/src_dependent_remote_index/common/connection.cc +++ /dev/null @@ -1,404 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - delete restart_queue; - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa, AtomicQueue** bb) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - restart_queue = *bb; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - if (message.type() == MessageProto::READ_RESULT || message.type() == MessageProto::DEPENDENT || message.type() == MessageProto::REMOTE_INDEX_REQUEST || message.type() == MessageProto::REMOTE_INDEX_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else if (message.type() == MessageProto::TXN_RESTART) { - restart_queue->Push(message); - }else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_dependent_remote_index/common/connection.h b/src_dependent_remote_index/common/connection.h deleted file mode 100755 index 5d601fa5..00000000 --- a/src_dependent_remote_index/common/connection.h +++ /dev/null @@ -1,201 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - Connection* NewConnection(const string& channel, AtomicQueue** aa, AtomicQueue** bb); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - AtomicQueue* restart_queue; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_dependent_remote_index/common/test_template.cc b/src_dependent_remote_index/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_dependent_remote_index/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_dependent_remote_index/common/testing.h b/src_dependent_remote_index/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_dependent_remote_index/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_dependent_remote_index/common/types.h b/src_dependent_remote_index/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_dependent_remote_index/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_dependent_remote_index/common/utils.h b/src_dependent_remote_index/common/utils.h deleted file mode 100755 index ea1021cb..00000000 --- a/src_dependent_remote_index/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_dependent_remote_index/common/zmq.hpp b/src_dependent_remote_index/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_dependent_remote_index/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_dependent_remote_index/deployment/Makefile.inc b/src_dependent_remote_index/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_dependent_remote_index/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_dependent_remote_index/deployment/addhosts.pl b/src_dependent_remote_index/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_dependent_remote_index/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_dependent_remote_index/deployment/cluster.cc b/src_dependent_remote_index/deployment/cluster.cc deleted file mode 100755 index c04f82b9..00000000 --- a/src_dependent_remote_index/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_dependent_remote_index/deployment/main.cc b/src_dependent_remote_index/deployment/main.cc deleted file mode 100755 index 36e5c53f..00000000 --- a/src_dependent_remote_index/deployment/main.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - - -unordered_map > index_; -vector mutex_for_index; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other; - do { - other = rand() % config_->all_nodes.size(); - } while (other == config_->this_node_id); - - if (dependent == 1) { - int index = rand() % INDEX_NUMBER; - *txn = microbenchmark.MicroTxnDependentMP(txn_id, config_->this_node_id, other, index); - } else { - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other); - } - } else { - // Single-partition txn. - if (dependent == 1) { - int index = rand() % INDEX_NUMBER; - *txn = microbenchmark.MicroTxnDependentSP(txn_id, index); - } else { - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - -//---------------------------Creat index------------------------------------ -int key; -int nparts = config.all_nodes.size(); -int key_start = config.all_nodes.size()*HOT; -int part = config.all_nodes.size() - config.this_node_id - 1; - -for(int i = 0; i < INDEX_NUMBER; i++) { - set myset; - vector myvector; - - int hotkey = part + nparts * i; - myvector.push_back(hotkey); - - for(int j = 0; j < 9; j++) { - do { - key = key_start + part + nparts * (rand() % ((1000000 - key_start)/nparts)); - } while(myset.count(key)); - myset.insert(key); - myvector.push_back(key); - } - index_[i] = myvector; -} -//------------------------------------------------------------------------- -mutex_for_index.resize(INDEX_NUMBER); -for (int i = 0; i < INDEX_NUMBER; i++) { - pthread_mutex_init(&mutex_for_index[i], NULL); -} - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, &multiplexer, client, storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_dependent_remote_index/deployment/makeconf.pl b/src_dependent_remote_index/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_dependent_remote_index/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_dependent_remote_index/deployment/portfile b/src_dependent_remote_index/deployment/portfile deleted file mode 100755 index c3166e3c..00000000 --- a/src_dependent_remote_index/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -55347 diff --git a/src_dependent_remote_index/deployment/run.pbs b/src_dependent_remote_index/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_dependent_remote_index/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_dependent_remote_index/deployment/test.conf b/src_dependent_remote_index/deployment/test.conf deleted file mode 100755 index 36bc2882..00000000 --- a/src_dependent_remote_index/deployment/test.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:1:12:128.36.232.12:51261 #rhino -node2=0:2:12:128.36.232.15:51262 # gator -node3=0:3:12:128.36.232.20:51261 #macaw -node4=0:4:12:128.36.232.19:51262 # peacock -node5=0:5:12:128.36.232.34:51261 # perch -node6=0:6:12:128.36.232.14:51262 # jaguar -node7=0:7:12:128.36.232.16:51261 # hippo -node8=0:8:12:128.36.232.39:51261 #viper -node9=0:9:12:128.36.232.7:51261 #hornet -node10=0:10:12:128.36.232.8:51261 #cricket -node11=0:11:12:128.36.232.28:51261 #termite -node12=0:12:12:128.36.232.5:51261 #cicada -node13=0:13:12:128.36.232.6:51261 #bumblebee -node14=0:14:12:128.36.232.44:51261 #frog -node15=0:15:12:128.36.232.50:51261 #tick -node16=0:16:12:128.36.232.42:51261 #turtle -node17=0:17:12:128.36.232.35:51261 #aphid -node18=0:18:12:128.36.232.41:51261 #chameleon -node19=0:19:12:128.36.232.38:51261 #rattlesnake -node20=0:20:12:128.36.232.45:51261 #scorpion -node21=0:21:12:128.36.232.29:51261 #cardinal -node22=0:22:12:128.36.232.24:51261 #swan -node23=0:23:12:128.36.232.27:51261 #monkey - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_dependent_remote_index/obj/.deps b/src_dependent_remote_index/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_dependent_remote_index/obj/common/configuration.d b/src_dependent_remote_index/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_dependent_remote_index/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_dependent_remote_index/paxos/Makefile.inc b/src_dependent_remote_index/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_dependent_remote_index/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_remote_index/paxos/paxos.cc b/src_dependent_remote_index/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_dependent_remote_index/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_dependent_remote_index/paxos/paxos.h b/src_dependent_remote_index/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_dependent_remote_index/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_dependent_remote_index/paxos/zookeeper.conf b/src_dependent_remote_index/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_dependent_remote_index/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_dependent_remote_index/proto/Makefile.inc b/src_dependent_remote_index/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_dependent_remote_index/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_dependent_remote_index/proto/message.proto b/src_dependent_remote_index/proto/message.proto deleted file mode 100755 index 17cd429a..00000000 --- a/src_dependent_remote_index/proto/message.proto +++ /dev/null @@ -1,70 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional string source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - DEPENDENT = 8; - TXN_RESTART = 9; - REMOTE_INDEX_REQUEST = 10; - REMOTE_INDEX_RESULT = 11; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; - - optional int32 restart = 2001; - - optional int32 index_number = 2002; - - optional string txn_id = 2003; -} - diff --git a/src_dependent_remote_index/proto/tpcc.proto b/src_dependent_remote_index/proto/tpcc.proto deleted file mode 100755 index 50088661..00000000 --- a/src_dependent_remote_index/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_dependent_remote_index/proto/tpcc_args.proto b/src_dependent_remote_index/proto/tpcc_args.proto deleted file mode 100755 index 248051c8..00000000 --- a/src_dependent_remote_index/proto/tpcc_args.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - - optional int32 order_number = 41; - - optional int32 lastest_order_number = 42; - optional int32 threshold = 51; -} diff --git a/src_dependent_remote_index/proto/txn.proto b/src_dependent_remote_index/proto/txn.proto deleted file mode 100755 index 31296fc5..00000000 --- a/src_dependent_remote_index/proto/txn.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; - - optional int32 index_number = 50; - optional int32 index_node = 51; - optional int32 other_node = 52; -} - diff --git a/src_dependent_remote_index/scheduler/Makefile.inc b/src_dependent_remote_index/scheduler/Makefile.inc deleted file mode 100755 index f5678dcc..00000000 --- a/src_dependent_remote_index/scheduler/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc \ - scheduler/serial_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_remote_index/scheduler/deterministic_lock_manager.cc b/src_dependent_remote_index/scheduler/deterministic_lock_manager.cc deleted file mode 100644 index 15d002e3..00000000 --- a/src_dependent_remote_index/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - deque* requests = it1->locksrequest; - - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - - } -} - diff --git a/src_dependent_remote_index/scheduler/deterministic_lock_manager.h b/src_dependent_remote_index/scheduler/deterministic_lock_manager.h deleted file mode 100755 index 2d10e4db..00000000 --- a/src_dependent_remote_index/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_dependent_remote_index/scheduler/deterministic_scheduler.cc b/src_dependent_remote_index/scheduler/deterministic_scheduler.cc deleted file mode 100755 index d863c57e..00000000 --- a/src_dependent_remote_index/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,557 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/zmq.hpp" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" -#include "applications/tpcc.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using zmq::socket_t; -using std::map; - -static void DeleteTxnPtr(void* data, void* hint) { free(data); } - -void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { - TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); - *txn_ptr = txn; - zmq::message_t msg(txn_ptr, sizeof(*txn_ptr), DeleteTxnPtr, NULL); - socket->send(msg); -} - -TxnProto* DeterministicScheduler::GetTxnPtr(socket_t* socket, - zmq::message_t* msg) { - if (!socket->recv(msg, ZMQ_NOBLOCK)) - return NULL; - TxnProto* txn = *reinterpret_cast(msg->data()); - return txn; -} - -//---------------------------Update index thread---------------------------------- -void* DeterministicScheduler::update_index(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - -int key; -int nparts = scheduler->configuration_->all_nodes.size(); -int key_start = scheduler->configuration_->all_nodes.size()*HOT; -int part = scheduler->configuration_->all_nodes.size() - scheduler->configuration_->this_node_id - 1; - - double volatility = VOLATILITY; - double total_updates = volatility * (double)INDEX_NUMBER; - double interval; - if (total_updates == 0) - interval = 1000; - else - interval = 1.0/total_updates; // time between consecutive updates - - double tick = GetTime(); // time of next update -int index = 0; - while (true) { - - if (GetTime() > tick) { - index = (index + 1) % INDEX_NUMBER; - set aa; - for (int i = 0; i < 10; i++) { - aa.insert(index_[index][i]); - } - do { - key = key_start + part + nparts * (rand() % ((1000000 - key_start)/nparts)); - } while (aa.count(key)); - - pthread_mutex_lock(&mutex_for_index[index]); - index_[index][4] = key; - pthread_mutex_unlock(&mutex_for_index[index]); - tick += interval; - } - - } - - return NULL; -} -//-------------------------------------------------------------------------------- - - - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - Connection* batch_connection, - Storage* storage, - const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - txns_queue = new AtomicQueue(); - done_queue = new AtomicQueue(); - - for (int i = 0; i < NUM_THREADS; i++) { - message_queues[i] = new AtomicQueue(); - } - - remote_index_queue_ = new AtomicQueue(); - - index_connection = batch_connection_->multiplexer()->NewConnection("remote_index", &remote_index_queue_); -Spin(2); - - // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(7, &cpuset); - pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); - pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, - reinterpret_cast(this)); - - -// pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { - string channel("scheduler"); - channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); - -pthread_attr_t attr; -pthread_attr_init(&attr); -CPU_ZERO(&cpuset); -if (i == 0 || i == 1) -CPU_SET(i, &cpuset); -else -CPU_SET(i+2, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, - reinterpret_cast( - new pair(i, this))); - } - - // Create update-index thread - pthread_attr_t attr; - pthread_attr_init(&attr); - CPU_ZERO(&cpuset); - CPU_SET(5, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&update_index_thread, &attr, update_index, reinterpret_cast(this)); - -} - -void UnfetchAll(Storage* storage, TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (StringToInt(txn->read_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_set(i)); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (StringToInt(txn->read_write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_write_set(i)); - for (int i = 0; i < txn->write_set_size(); i++) - if (StringToInt(txn->write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->write_set(i)); -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - int thread = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - unordered_map active_txns; - unordered_map pending_txns; - - bool got_message; - - // Begin main loop. - MessageProto message; - while (true) { - got_message = scheduler->remote_index_queue_->Pop(&message); - if (got_message == true) { - int index_number = message.index_number(); - - pthread_mutex_lock(&mutex_for_index[index_number]); - vector aa = index_[index_number]; - pthread_mutex_unlock(&mutex_for_index[index_number]); - - for (int i = 0; i < 10; i++) { - message.add_keys(IntToString(aa[i])); - } - - message.set_type(MessageProto::REMOTE_INDEX_RESULT); - message.set_destination_channel(message.source_channel()); - message.set_destination_node(scheduler->configuration_->all_nodes.size() - scheduler->configuration_->this_node_id - 1); - - scheduler->index_connection->Send1(message); - } else { - - got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - if (message.type() == MessageProto::READ_RESULT) { - // Remote read result. - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - // Execute and clean up. - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - delete manager; - - scheduler->thread_connections_[thread]->UnlinkChannel(IntToString(txn->txn_id())); - active_txns.erase(message.destination_channel()); - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } - } else if (message.type() == MessageProto::DEPENDENT) { - int restart = message.restart(); - if (restart == 0) { - TxnProto* txn = pending_txns[message.destination_channel()]; - pending_txns.erase(message.destination_channel()); - // Create manager. - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } else { - active_txns[IntToString(txn->txn_id())] = manager; - } - } else { - TxnProto* txn = pending_txns[message.destination_channel()]; - pending_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(IntToString(txn->txn_id())); - txn->set_status(TxnProto::ABORTED); - scheduler->done_queue->Push(txn); - } - } else if (message.type() == MessageProto::REMOTE_INDEX_RESULT) { - - TxnProto* txn = pending_txns[message.destination_channel()]; - assert(message.keys_size() == 10); - int restart = 0; - - if (txn->txn_type() == 3) { - for (int i = 0; i < message.keys_size(); i++) { - if (message.keys(i) != txn->read_write_set(i)) { - restart = 1; - break; - } - } - } else if (txn->txn_type() == 4) { - int j = 0; - for (int i = 0; i < message.keys_size(); i++) { - if (message.keys(i) != txn->read_write_set(j + 5)) { - restart = 1; - break; - } - j++; - if (j >= 5) { - break; - } - } - } - - // Send the restart information to the other node - if (txn->txn_type() == 4) { - MessageProto message; - message.set_type(MessageProto::DEPENDENT); - if (restart == 0) { - message.set_restart(0); - } else { - message.set_restart(1); - } - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->other_node()); - scheduler->thread_connections_[thread]->Send1(message); - } - - // Execute the txn or abort it - if (restart == 0) { - pending_txns.erase(message.destination_channel()); - // Create manager. - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } else { - active_txns[IntToString(txn->txn_id())] = manager; - } - - } else { - pending_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(IntToString(txn->txn_id())); - txn->set_status(TxnProto::ABORTED); - scheduler->done_queue->Push(txn); - } - - } - } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { - int restart = 0; - if (txn->txn_type() == 3) { - MessageProto message; - message.set_type(MessageProto::REMOTE_INDEX_REQUEST); - message.set_destination_channel("remote_index"); - message.set_destination_node(scheduler->configuration_->all_nodes.size() - scheduler->configuration_->this_node_id - 1); - message.set_index_number(txn->index_number()); - message.set_source_node(scheduler->configuration_->this_node_id); - message.set_source_channel(IntToString(txn->txn_id())); - message.set_txn_id(IntToString(txn->txn_id())); - scheduler->thread_connections_[thread]->Send1(message); - - scheduler->thread_connections_[thread]->LinkChannel(IntToString(txn->txn_id())); - pending_txns[IntToString(txn->txn_id())] = txn; - restart = 2; - } - - if (txn->txn_type() == 4) { - if (txn->index_node() == scheduler->configuration_->this_node_id) { - - MessageProto message; - message.set_type(MessageProto::REMOTE_INDEX_REQUEST); - message.set_destination_channel("remote_index"); - message.set_destination_node(scheduler->configuration_->all_nodes.size() - scheduler->configuration_->this_node_id - 1); - message.set_index_number(txn->index_number()); - message.set_source_node(scheduler->configuration_->this_node_id); - message.set_source_channel(IntToString(txn->txn_id())); - message.set_txn_id(IntToString(txn->txn_id())); - scheduler->thread_connections_[thread]->Send1(message); - scheduler->thread_connections_[thread]->LinkChannel(IntToString(txn->txn_id())); - pending_txns[IntToString(txn->txn_id())] = txn; - restart = 2; - } else { - scheduler->thread_connections_[thread]->LinkChannel(IntToString(txn->txn_id())); - pending_txns[IntToString(txn->txn_id())] = txn; - restart = 2; - } - } - - if (restart == 0) { - // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } else { - scheduler->thread_connections_[thread]->LinkChannel(IntToString(txn->txn_id())); - active_txns[IntToString(txn->txn_id())] = manager; - } - } else if (restart == 1) { - txn->set_status(TxnProto::ABORTED); - scheduler->done_queue->Push(txn); - } - } - } - } - } - return NULL; -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -MessageProto* GetBatch(int batch_id, Connection* connection) { - if (batches.count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[batch_id]; - batches.erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - - // Run main loop. - MessageProto message; - MessageProto* batch_message = NULL; - int txns = 0; - double time = GetTime(); - int executing_txns = 0; - int pending_txns = 0; - int batch_offset = 0; - int batch_number = 0; -int test = 0; -int abort_number = 0;; - while (true) { -//if (scheduler->configuration_->this_node_id ==1) -//printf("LM thread running\n"); - TxnProto* done_txn; - bool got_it = scheduler->done_queue->Pop(&done_txn); - if (got_it == true) { - // We have received a finished transaction back, release the lock - scheduler->lock_manager_->Release(done_txn); - executing_txns--; - - if (done_txn->status() == TxnProto::ABORTED) { - - if (done_txn->txn_type() == 3) { -abort_number++; - done_txn->set_status(TxnProto::ACTIVE); - done_txn->clear_read_write_set(); - done_txn->clear_readers(); - done_txn->clear_writers(); - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(scheduler->configuration_->this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - done_txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->batch_connection_->Send(restart); - delete done_txn; - } else if (done_txn->txn_type() == 4 && done_txn->index_node() == scheduler->configuration_->this_node_id) { -abort_number++; - done_txn->set_status(TxnProto::ACTIVE); - done_txn->clear_readers(); - done_txn->clear_writers(); - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(scheduler->configuration_->this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - done_txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->batch_connection_->Send(restart); - delete done_txn; - } else { - delete done_txn; - } - } else { - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) { - txns++; - } - delete done_txn; - } - - } else { - // Have we run out of txns in our batch? Let's get some new ones. - if (batch_message == NULL) { - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Done with current batch, get next. - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { - if (batch_offset >= batch_message->data_size()) { - // Oops we ran out of txns in this batch. Stop adding txns for now. - break; - } - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - scheduler->lock_manager_->Lock(txn); - pending_txns++; - } - - } - } - - // Start executing any and all ready transactions to get them off our plate - while (!scheduler->ready_txns_->empty()) { - TxnProto* txn = scheduler->ready_txns_->front(); - scheduler->ready_txns_->pop_front(); - pending_txns--; - executing_txns++; - - scheduler->txns_queue->Push(txn); - //scheduler->SendTxnPtr(scheduler->requests_out_, txn); - - } - - // Report throughput. - if (GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout << "Completed " << (static_cast(txns) / total_time) - << " txns/sec, " - << abort_number<< " transaction restart , " - << test << " second, " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - abort_number = 0; - test++; - } - } - return NULL; -} diff --git a/src_dependent_remote_index/scheduler/deterministic_scheduler.h b/src_dependent_remote_index/scheduler/deterministic_scheduler.h deleted file mode 100755 index 82d6549b..00000000 --- a/src_dependent_remote_index/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,100 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include - -#include "scheduler/scheduler.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; - -namespace zmq { -class socket_t; -class message_t; -} -using zmq::socket_t; - -class Configuration; -class Connection; -class DeterministicLockManager; -class Storage; -class TxnProto; - -#define NUM_THREADS 3 -// #define PREFETCHING - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - static void* LockManagerThread(void* arg); - - static void* update_index(void* arg); - - void SendTxnPtr(socket_t* socket, TxnProto* txn); - TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; - - pthread_t lock_manager_thread_; - // Connection for receiving txn batches from sequencer. - Connection* batch_connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Application currently being run. - const Application* application_; - - // The per-node lock manager tracks what transactions have temporary ownership - // of what database objects, allowing the scheduler to track LOCAL conflicts - // and enforce equivalence to transaction orders. - DeterministicLockManager* lock_manager_; - - // Queue of transaction ids of transactions that have acquired all locks that - // they have requested. - std::deque* ready_txns_; - - // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; - - AtomicQueue* txns_queue; - AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - - AtomicQueue* remote_index_queue_; - - Connection* index_connection; - - pthread_t update_index_thread; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_dependent_remote_index/scheduler/lock_manager.h b/src_dependent_remote_index/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_dependent_remote_index/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_dependent_remote_index/scheduler/scheduler.h b/src_dependent_remote_index/scheduler/scheduler.h deleted file mode 100755 index d93465fa..00000000 --- a/src_dependent_remote_index/scheduler/scheduler.h +++ /dev/null @@ -1,19 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} - -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_dependent_remote_index/scheduler/serial_scheduler.cc b/src_dependent_remote_index/scheduler/serial_scheduler.cc deleted file mode 100755 index 1d2c9bfe..00000000 --- a/src_dependent_remote_index/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_dependent_remote_index/scheduler/serial_scheduler.h b/src_dependent_remote_index/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_dependent_remote_index/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_dependent_remote_index/sequencer/Makefile.inc b/src_dependent_remote_index/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_dependent_remote_index/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_remote_index/sequencer/sequencer.cc b/src_dependent_remote_index/sequencer/sequencer.cc deleted file mode 100755 index 1edb36e5..00000000 --- a/src_dependent_remote_index/sequencer/sequencer.cc +++ /dev/null @@ -1,441 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, ConnectionMultiplexer* multiplexer, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), multiplexer_(multiplexer), - client_(client), storage_(storage), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - // Start Sequencer main loops running in background thread. - -message_queues = new AtomicQueue(); -restart_queues = new AtomicQueue(); -connection_ = multiplexer_->NewConnection("sequencer", &message_queues, &restart_queues); -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); - delete message_queues; -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -#ifdef PREFETCHING -double PrefetchAll(Storage* storage, TxnProto* txn) { - double max_wait_time = 0; - double wait_time = 0; - for (int i = 0; i < txn->read_set_size(); i++) { - storage->Prefetch(txn->read_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - storage->Prefetch(txn->read_write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->write_set_size(); i++) { - storage->Prefetch(txn->write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) - prefetch_cold[txn->txn_id() / SAMPLE_RATE] = max_wait_time; -#endif - return max_wait_time; -} -#endif - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - -#ifdef PREFETCHING - multimap fetching_txns; -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - std::cout << "Starting sequencer.\n" << std::flush; - -unordered_map pending_txns; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); - - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (txn_id_offset < MAX_BATCH_SIZE && batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - - bool got_message = message_queues->Pop(&message); - if(got_message == true) { - if (message.type() == MessageProto::REMOTE_INDEX_RESULT) { - txn = pending_txns[message.txn_id()]; - pending_txns.erase(message.txn_id()); - if (txn->txn_type() == 3) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - } - } else if (txn->txn_type() == 4) { - int j = 0; - if (txn->read_write_set_size() == 5) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - j++; - if (j >= 5) { - break; - } - } - } else if (txn->read_write_set_size() == 10) { - for (int i = 0; i < message.keys_size(); i++) { - txn->set_read_write_set(5 + j, message.keys(i)); - j++; - if (j >= 5) { - break; - } - } - } - - } - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } else { - - if (rand() % 100 < DEPENDENT_PERCENT) { - got_message = restart_queues->Pop(&message); - if (got_message == true) { - assert(message.type() == MessageProto::TXN_RESTART); - txn = new TxnProto(); - txn->ParseFromString(message.data(0)); - txn->set_txn_id(batch_number * MAX_BATCH_SIZE + txn_id_offset); - txn_id_offset++; - MessageProto message; - message.set_type(MessageProto::REMOTE_INDEX_REQUEST); - message.set_destination_channel("remote_index"); - message.set_destination_node(configuration_->all_nodes.size() - configuration_->this_node_id - 1); - message.set_index_number(txn->index_number()); - message.set_source_node(configuration_->this_node_id); - message.set_source_channel("sequencer"); - message.set_txn_id(IntToString(txn->txn_id())); - connection_->Send1(message); - - pending_txns[IntToString(txn->txn_id())] = txn; - } else { - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset, 1); - - MessageProto message; - message.set_type(MessageProto::REMOTE_INDEX_REQUEST); - message.set_destination_channel("remote_index"); - message.set_destination_node(configuration_->all_nodes.size() - configuration_->this_node_id - 1); - message.set_index_number(txn->index_number()); - message.set_source_node(configuration_->this_node_id); - message.set_source_channel("sequencer"); - message.set_txn_id(IntToString(txn->txn_id())); - connection_->Send1(message); - - pending_txns[IntToString(txn->txn_id())] = txn; - txn_id_offset++; - } - } else { - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset, 0); - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } - } - } else if (txn_id_offset >= MAX_BATCH_SIZE && batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - - bool got_message = message_queues->Pop(&message); - if(got_message == true) { - if (message.type() == MessageProto::REMOTE_INDEX_RESULT) { - txn = pending_txns[message.txn_id()]; - pending_txns.erase(message.txn_id()); - if (txn->txn_type() == 3) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - } - } else if (txn->txn_type() == 4) { - int j = 0; - if (txn->read_write_set_size() == 5) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - j++; - if (j >= 5) { - break; - } - } - } else if (txn->read_write_set_size() == 10) { - for (int i = 0; i < message.keys_size(); i++) { - txn->set_read_write_set(5 + j, message.keys(i)); - j++; - if (j >= 5) { - break; - } - } - } - - } -//test++; - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } - } - } - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - - // Set up batch messages for each system node. - map batches; - for (map::iterator it = configuration_->all_nodes.begin(); - it != configuration_->all_nodes.end(); ++it) { - batches[it->first].set_destination_channel("scheduler_"); - batches[it->first].set_destination_node(it->first); - batches[it->first].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif - - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - } while (!got_batch); -#endif -//assert(batch_message.data_size() > 0); - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); -#ifdef LATENCY_TEST - if (txn.txn_id() % SAMPLE_RATE == 0) - watched_txn = txn.txn_id(); -#endif - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - - bytes txn_data; - txn.SerializeToString(&txn_data); - - // Compute union of 'readers' and 'writers' (store in 'readers'). - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - // Insert txn into appropriate batches. - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - batches[*it].add_data(txn_data); - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - -#ifdef LATENCY_TEST - if (watched_txn != -1) { - sequencer_send[watched_txn] = GetTime(); - watched_txn = -1; - } -#endif - - // Report output. - if (GetTime() > time + 1) { -#ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; -#endif - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_dependent_remote_index/sequencer/sequencer.h b/src_dependent_remote_index/sequencer/sequencer.h deleted file mode 100755 index abb3dbb9..00000000 --- a/src_dependent_remote_index/sequencer/sequencer.h +++ /dev/null @@ -1,138 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include -#include - -#include "common/types.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "common/connection.h" - -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 155 - -#define SAMPLES 100000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST - -using std::set; -using std::string; -using std::queue; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -#define HOT 10000 - -//----------------------------------- -#define INDEX_NUMBER 10000 -#define DEPENDENT_PERCENT 20 -#define VOLATILITY 100 -extern unordered_map > index_; -extern vector mutex_for_index; -//----------------------------------- - - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, ConnectionMultiplexer* multiplexer, Client* client, - Storage* storage); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - ConnectionMultiplexer* multiplexer_; - - // Client from which to get incoming txns. - Client* client_; - - // Pointer to this node's storage object, for prefetching. - Storage* storage_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; - - AtomicQueue* message_queues; - AtomicQueue* restart_queues; - -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_dependent_remote_index/tests/collapsed_versioned_storage_test.cc b/src_dependent_remote_index/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_dependent_remote_index/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_dependent_remote_index/tests/configuration_test.cc b/src_dependent_remote_index/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_dependent_remote_index/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_dependent_remote_index/tests/connection_test.cc b/src_dependent_remote_index/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_dependent_remote_index/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_dependent_remote_index/tests/deterministic_lock_manager_test.cc b/src_dependent_remote_index/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_dependent_remote_index/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_dependent_remote_index/tests/deterministic_scheduler_test.cc b/src_dependent_remote_index/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_dependent_remote_index/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_dependent_remote_index/tests/fetching_storage_test.cc b/src_dependent_remote_index/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_dependent_remote_index/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_dependent_remote_index/tests/microbenchmark_test.cc b/src_dependent_remote_index/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_dependent_remote_index/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_dependent_remote_index/tests/paxos_test.cc b/src_dependent_remote_index/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_dependent_remote_index/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_dependent_remote_index/tests/profile_test.cc b/src_dependent_remote_index/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_dependent_remote_index/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_dependent_remote_index/tests/sequencer_test.cc b/src_dependent_remote_index/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_dependent_remote_index/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_dependent_remote_index/tests/serial_scheduler_test.cc b/src_dependent_remote_index/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_dependent_remote_index/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_dependent_remote_index/tests/simple_storage_test.cc b/src_dependent_remote_index/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_dependent_remote_index/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_dependent_remote_index/tests/storage_manager_test.cc b/src_dependent_remote_index/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_dependent_remote_index/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_dependent_remote_index/tests/tpcc_test.cc b/src_dependent_remote_index/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_dependent_remote_index/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_dependent_remote_index/tests/types_test.cc b/src_dependent_remote_index/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_dependent_remote_index/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/Makefile b/src_dependent_variable_sized_reconnaissance_phases/Makefile deleted file mode 100755 index e3e6e17d..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_dependent_variable_sized_reconnaissance_phases/Makefile.template b/src_dependent_variable_sized_reconnaissance_phases/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/application.h b/src_dependent_variable_sized_reconnaissance_phases/applications/application.h deleted file mode 100755 index 047e8513..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, - Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.cc b/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.cc deleted file mode 100755 index 12600ad3..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.cc +++ /dev/null @@ -1,188 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnDependentSP(int64 txn_id, int64 index_number) { - TxnProto* txn = new TxnProto(); - txn->set_txn_id(txn_id); - txn->set_index_number(index_number); - txn->set_txn_type(MICROTXN_DEPENDENT_SP); - return txn; -} - -// Create a dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnDependentMP(int64 txn_id, int part1, int part2, int64 index_number) { - TxnProto* txn = new TxnProto(); - txn->set_txn_id(txn_id); - txn->set_index_number(index_number); - txn->set_index_node(part1); - txn->set_other_node(part2); - txn->set_txn_type(MICROTXN_DEPENDENT_MP); - - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey2)); - - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - return txn; -} - -// Create a non-dependent single-partition transaction -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// Create a non-dependent multi-partition transaction -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { - assert(part1 != part2 || nparts == 1); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - int x = 1; - for(int i = 0; i < 10000; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, - Configuration* conf) const { - for (int i = 0; i < nparts*kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { -#ifdef PREFETCHING - if (i % 10000 == 0) - std::cout << i << std::endl; - storage->Prefetch(IntToString(i), &wait_time); - storage->PutObject(IntToString(i), new Value(IntToString(i))); - if (i > COLD_CUTOFF) { - storage->Unfetch(IntToString(i)); - if (i % 10 == 0) - std::cout << i << std::endl; - } -#else - storage->PutObject(IntToString(i), new Value(IntToString(i))); -#endif - } - } -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.h b/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.h deleted file mode 100755 index 9e0afc72..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/microbenchmark.h +++ /dev/null @@ -1,59 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - MICROTXN_DEPENDENT_SP = 3, - MICROTXN_DEPENDENT_MP = 4, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2); - TxnProto* MicroTxnDependentSP(int64 txn_id, int64 index_number); - TxnProto* MicroTxnDependentMP(int64 txn_id, int part1, int part2, int64 index_number); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.cc b/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.h b/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/backend/Makefile.inc deleted file mode 100644 index edcf67bd..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/Makefile.inc +++ /dev/null @@ -1,28 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/checkpointable_storage.cc \ - backend/collapsed_versioned_storage.cc \ - backend/fetching_storage.cc \ - backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.cc b/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.cc deleted file mode 100644 index aa0f99f9..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.h b/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.h deleted file mode 100644 index 249a40d6..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.cc b/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.cc deleted file mode 100644 index f5514372..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.h b/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.h deleted file mode 100644 index 273dc04c..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.cc b/src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.cc deleted file mode 100644 index 46237c8f..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/storage.h b/src_dependent_variable_sized_reconnaissance_phases/backend/storage.h deleted file mode 100644 index fa2e4efb..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/storage.h +++ /dev/null @@ -1,49 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.cc b/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.cc deleted file mode 100644 index 7de4ff40..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == configuration_->this_node_id) - reader = true; - } - - if (reader) { - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != configuration_->this_node_id) { - message.set_destination_node(txn->writers(i)); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == configuration_->this_node_id) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.h b/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.h deleted file mode 100644 index a374017a..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/storage_manager.h +++ /dev/null @@ -1,87 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_dependent_variable_sized_reconnaissance_phases/backend/versioned_storage.h b/src_dependent_variable_sized_reconnaissance_phases/backend/versioned_storage.h deleted file mode 100644 index 24c6223b..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/configuration.cc b/src_dependent_variable_sized_reconnaissance_phases/common/configuration.cc deleted file mode 100755 index b07312ee..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/configuration.cc +++ /dev/null @@ -1,119 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/configuration.h b/src_dependent_variable_sized_reconnaissance_phases/common/configuration.h deleted file mode 100755 index 43a1289e..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/configuration.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/types.h" - - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - -struct Node { - // Globally unique node identifier. - int node_id; - int replica_id; - int partition_id; - - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/configuration_test.conf b/src_dependent_variable_sized_reconnaissance_phases/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/connection.cc b/src_dependent_variable_sized_reconnaissance_phases/common/connection.cc deleted file mode 100755 index 8466c191..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/connection.cc +++ /dev/null @@ -1,404 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - delete restart_queue; - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa, AtomicQueue** bb) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - restart_queue = *bb; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - if (message.type() == MessageProto::READ_RESULT || message.type() == MessageProto::DEPENDENT || message.type() == MessageProto::REMOTE_INDEX_REQUEST || message.type() == MessageProto::REMOTE_INDEX_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else if (message.type() == MessageProto::TXN_RESTART) { - restart_queue->Push(message); - }else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/connection.h b/src_dependent_variable_sized_reconnaissance_phases/common/connection.h deleted file mode 100755 index 5d601fa5..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/connection.h +++ /dev/null @@ -1,201 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - Connection* NewConnection(const string& channel, AtomicQueue** aa, AtomicQueue** bb); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - AtomicQueue* restart_queue; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/test_template.cc b/src_dependent_variable_sized_reconnaissance_phases/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/testing.h b/src_dependent_variable_sized_reconnaissance_phases/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/types.h b/src_dependent_variable_sized_reconnaissance_phases/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/utils.h b/src_dependent_variable_sized_reconnaissance_phases/common/utils.h deleted file mode 100755 index ea1021cb..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_dependent_variable_sized_reconnaissance_phases/common/zmq.hpp b/src_dependent_variable_sized_reconnaissance_phases/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/addhosts.pl b/src_dependent_variable_sized_reconnaissance_phases/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/cluster.cc b/src_dependent_variable_sized_reconnaissance_phases/deployment/cluster.cc deleted file mode 100755 index c04f82b9..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/main.cc b/src_dependent_variable_sized_reconnaissance_phases/deployment/main.cc deleted file mode 100755 index b59e34f1..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/main.cc +++ /dev/null @@ -1,218 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - - -unordered_map > index_; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other; - do { - other = rand() % config_->all_nodes.size(); - } while (other == config_->this_node_id); - - if (dependent == 1) { - int index = rand() % INDEX_NUMBER; - *txn = microbenchmark.MicroTxnDependentMP(txn_id, config_->this_node_id, other, index); - } else { - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other); - } - } else { - // Single-partition txn. - if (dependent == 1) { - int index = rand() % INDEX_NUMBER; - *txn = microbenchmark.MicroTxnDependentSP(txn_id, index); - } else { - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - -//---------------------------Creat index------------------------------------ -int key; -int nparts = config.all_nodes.size(); -int key_start = config.all_nodes.size()*HOT; -int part = config.this_node_id; - -for(int i = 0; i < INDEX_NUMBER; i++) { - set myset; - vector myvector; - - int hotkey = part + nparts * i; - myvector.push_back(hotkey); - - for(int j = 0; j < 9; j++) { - do { - key = key_start + part + nparts * (rand() % ((1000000 - key_start)/nparts)); - } while(myset.count(key)); - myset.insert(key); - myvector.push_back(key); - } - index_[i] = myvector; -} -//------------------------------------------------------------------------- - - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, &multiplexer, client, storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/makeconf.pl b/src_dependent_variable_sized_reconnaissance_phases/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/portfile b/src_dependent_variable_sized_reconnaissance_phases/deployment/portfile deleted file mode 100755 index 1958cd06..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -55350 diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/run.pbs b/src_dependent_variable_sized_reconnaissance_phases/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_dependent_variable_sized_reconnaissance_phases/deployment/test.conf b/src_dependent_variable_sized_reconnaissance_phases/deployment/test.conf deleted file mode 100755 index 36bc2882..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/deployment/test.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:1:12:128.36.232.12:51261 #rhino -node2=0:2:12:128.36.232.15:51262 # gator -node3=0:3:12:128.36.232.20:51261 #macaw -node4=0:4:12:128.36.232.19:51262 # peacock -node5=0:5:12:128.36.232.34:51261 # perch -node6=0:6:12:128.36.232.14:51262 # jaguar -node7=0:7:12:128.36.232.16:51261 # hippo -node8=0:8:12:128.36.232.39:51261 #viper -node9=0:9:12:128.36.232.7:51261 #hornet -node10=0:10:12:128.36.232.8:51261 #cricket -node11=0:11:12:128.36.232.28:51261 #termite -node12=0:12:12:128.36.232.5:51261 #cicada -node13=0:13:12:128.36.232.6:51261 #bumblebee -node14=0:14:12:128.36.232.44:51261 #frog -node15=0:15:12:128.36.232.50:51261 #tick -node16=0:16:12:128.36.232.42:51261 #turtle -node17=0:17:12:128.36.232.35:51261 #aphid -node18=0:18:12:128.36.232.41:51261 #chameleon -node19=0:19:12:128.36.232.38:51261 #rattlesnake -node20=0:20:12:128.36.232.45:51261 #scorpion -node21=0:21:12:128.36.232.29:51261 #cardinal -node22=0:22:12:128.36.232.24:51261 #swan -node23=0:23:12:128.36.232.27:51261 #monkey - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_dependent_variable_sized_reconnaissance_phases/obj/.deps b/src_dependent_variable_sized_reconnaissance_phases/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_dependent_variable_sized_reconnaissance_phases/obj/common/configuration.d b/src_dependent_variable_sized_reconnaissance_phases/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_dependent_variable_sized_reconnaissance_phases/paxos/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.cc b/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.h b/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/paxos/zookeeper.conf b/src_dependent_variable_sized_reconnaissance_phases/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_dependent_variable_sized_reconnaissance_phases/proto/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_dependent_variable_sized_reconnaissance_phases/proto/message.proto b/src_dependent_variable_sized_reconnaissance_phases/proto/message.proto deleted file mode 100755 index 17cd429a..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/proto/message.proto +++ /dev/null @@ -1,70 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional string source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - DEPENDENT = 8; - TXN_RESTART = 9; - REMOTE_INDEX_REQUEST = 10; - REMOTE_INDEX_RESULT = 11; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; - - optional int32 restart = 2001; - - optional int32 index_number = 2002; - - optional string txn_id = 2003; -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc.proto b/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc.proto deleted file mode 100755 index 50088661..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc_args.proto b/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc_args.proto deleted file mode 100755 index 248051c8..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/proto/tpcc_args.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - - optional int32 order_number = 41; - - optional int32 lastest_order_number = 42; - optional int32 threshold = 51; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/proto/txn.proto b/src_dependent_variable_sized_reconnaissance_phases/proto/txn.proto deleted file mode 100755 index 31296fc5..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/proto/txn.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; - - optional int32 index_number = 50; - optional int32 index_node = 51; - optional int32 other_node = 52; -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/scheduler/Makefile.inc deleted file mode 100755 index f5678dcc..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc \ - scheduler/serial_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.cc b/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.cc deleted file mode 100644 index 15d002e3..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - deque* requests = it1->locksrequest; - - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - - } -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.h b/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.h deleted file mode 100755 index 1bba5425..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.cc b/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.cc deleted file mode 100755 index 367e07bf..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,384 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/zmq.hpp" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" -#include "applications/tpcc.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using zmq::socket_t; -using std::map; - -static void DeleteTxnPtr(void* data, void* hint) { free(data); } - -void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { - TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); - *txn_ptr = txn; - zmq::message_t msg(txn_ptr, sizeof(*txn_ptr), DeleteTxnPtr, NULL); - socket->send(msg); -} - -TxnProto* DeterministicScheduler::GetTxnPtr(socket_t* socket, - zmq::message_t* msg) { - if (!socket->recv(msg, ZMQ_NOBLOCK)) - return NULL; - TxnProto* txn = *reinterpret_cast(msg->data()); - return txn; -} - -//---------------------------Update index thread---------------------------------- -void* DeterministicScheduler::update_index(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - -int key; -int nparts = scheduler->configuration_->all_nodes.size(); -int key_start = scheduler->configuration_->all_nodes.size()*HOT; -int part = scheduler->configuration_->this_node_id; - - double volatility = VOLATILITY; - double total_updates = volatility * (double)INDEX_NUMBER; - double interval; - if (total_updates == 0) - interval = 1000; - else - interval = 1.0/total_updates; // time between consecutive updates - - double tick = GetTime(); // time of next update -int index = 0; - while (true) { - - if (GetTime() > tick) { - index = (index + 1) % INDEX_NUMBER; - set aa; - for (int i = 0; i < 10; i++) { - aa.insert(index_[index][i]); - } - do { - key = key_start + part + nparts * (rand() % ((1000000 - key_start)/nparts)); - } while (aa.count(key)); - //index_[index][4] = key; - tick += interval; - } - - } - - return NULL; -} -//-------------------------------------------------------------------------------- - - - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - Connection* batch_connection, - Storage* storage, - const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - txns_queue = new AtomicQueue(); - done_queue = new AtomicQueue(); - - for (int i = 0; i < NUM_THREADS; i++) { - message_queues[i] = new AtomicQueue(); - } - - remote_index_queue_ = new AtomicQueue(); - - index_connection = batch_connection_->multiplexer()->NewConnection("remote_index", &remote_index_queue_); -//Spin(10); - - // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(7, &cpuset); - pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); - pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, - reinterpret_cast(this)); - - -// pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { - string channel("scheduler"); - channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); - -pthread_attr_t attr; -pthread_attr_init(&attr); -CPU_ZERO(&cpuset); -if (i == 0 || i == 1) -CPU_SET(i, &cpuset); -else -CPU_SET(i+2, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, - reinterpret_cast( - new pair(i, this))); - } - - // Create update-index thread - pthread_attr_t attr; - pthread_attr_init(&attr); - CPU_ZERO(&cpuset); - CPU_SET(5, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&update_index_thread, &attr, update_index, reinterpret_cast(this)); - -} - -void UnfetchAll(Storage* storage, TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (StringToInt(txn->read_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_set(i)); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (StringToInt(txn->read_write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_write_set(i)); - for (int i = 0; i < txn->write_set_size(); i++) - if (StringToInt(txn->write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->write_set(i)); -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - int thread = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - unordered_map active_txns; - unordered_map pending_txns; - - bool got_message; - - // Begin main loop. - MessageProto message; - while (true) { - got_message = scheduler->remote_index_queue_->Pop(&message); - if (got_message == true) { - int x = 1; - for(int i = 0; i < 9000; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - int index_number = message.index_number(); - vector aa = index_[index_number]; - for (int i = 0; i < 10; i++) { - message.add_keys(IntToString(aa[i])); - } - - message.set_type(MessageProto::REMOTE_INDEX_RESULT); - message.set_destination_channel(message.source_channel()); - message.set_destination_node(scheduler->configuration_->this_node_id); - - scheduler->index_connection->Send(message); - } else { - - got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - if (message.type() == MessageProto::READ_RESULT) { - // Remote read result. - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - // Execute and clean up. - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - delete manager; - - scheduler->thread_connections_[thread]->UnlinkChannel(IntToString(txn->txn_id())); - active_txns.erase(message.destination_channel()); - // Respond to scheduler; - scheduler->done_queue->Push(txn); - } - } - } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { - int restart = 0; - if (txn->txn_type() == 3) { - - int index_number = txn->index_number(); - vector aa = index_[index_number]; - for (int i = 0; i < 10; i++) { - if (aa[i] != StringToInt(txn->read_write_set(i))) { - restart = 1; - break; - } - } - } - - if (restart != 0) { - printf("********************************************************* FUCK FUCK FUCK *******************************************************\n"); - } - - if (restart == 0) { - int x = 1; - for(int i = 0; i < 10000; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - - scheduler->done_queue->Push(txn); - } - } - } - } - } - return NULL; -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -MessageProto* GetBatch(int batch_id, Connection* connection) { - if (batches.count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[batch_id]; - batches.erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - - // Run main loop. - MessageProto message; - MessageProto* batch_message = NULL; - int txns = 0; - double time = GetTime(); - int executing_txns = 0; - int pending_txns = 0; - int batch_offset = 0; - int batch_number = 0; -int test = 0; - while (true) { -//if (scheduler->configuration_->this_node_id ==1) -//printf("LM thread running\n"); - TxnProto* done_txn; - bool got_it = scheduler->done_queue->Pop(&done_txn); - if (got_it == true) { - // We have received a finished transaction back, release the lock - - scheduler->lock_manager_->Release(done_txn); - executing_txns--; - - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) { - txns++; - } - delete done_txn; - } else { - // Have we run out of txns in our batch? Let's get some new ones. - if (batch_message == NULL) { - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Done with current batch, get next. - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { - if (batch_offset >= batch_message->data_size()) { - // Oops we ran out of txns in this batch. Stop adding txns for now. - break; - } - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - scheduler->lock_manager_->Lock(txn); - pending_txns++; - } - - } - } - - // Start executing any and all ready transactions to get them off our plate - while (!scheduler->ready_txns_->empty()) { - TxnProto* txn = scheduler->ready_txns_->front(); - scheduler->ready_txns_->pop_front(); - pending_txns--; - executing_txns++; - - scheduler->txns_queue->Push(txn); - } - - // Report throughput. - if (GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout << "Completed " << (static_cast(txns) / total_time) - << " txns/sec, " - << test << " second, " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - test++; - } - } - return NULL; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.h b/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.h deleted file mode 100755 index 51c204c7..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,101 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include - -#include "scheduler/scheduler.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; - -namespace zmq { -class socket_t; -class message_t; -} -using zmq::socket_t; - -class Configuration; -class Connection; -class DeterministicLockManager; -class Storage; -class TxnProto; - -#define NUM_THREADS 3 -// #define PREFETCHING - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, Connection* batch_connection, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - static void* LockManagerThread(void* arg); - - static void* update_index(void* arg); - - void SendTxnPtr(socket_t* socket, TxnProto* txn); - TxnProto* GetTxnPtr(socket_t* socket, zmq::message_t* msg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[NUM_THREADS]; - Connection* thread_connections_[NUM_THREADS]; - - pthread_t lock_manager_thread_; - // Connection for receiving txn batches from sequencer. - Connection* batch_connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Application currently being run. - const Application* application_; - - // The per-node lock manager tracks what transactions have temporary ownership - // of what database objects, allowing the scheduler to track LOCAL conflicts - // and enforce equivalence to transaction orders. - DeterministicLockManager* lock_manager_; - - // Queue of transaction ids of transactions that have acquired all locks that - // they have requested. - std::deque* ready_txns_; - - // Sockets for communication between main scheduler thread and worker threads. -// socket_t* requests_out_; -// socket_t* requests_in_; -// socket_t* responses_out_[NUM_THREADS]; -// socket_t* responses_in_; - - AtomicQueue* txns_queue; - AtomicQueue* done_queue; - - AtomicQueue* message_queues[NUM_THREADS]; - - AtomicQueue* remote_index_queue_; - - Connection* index_connection; - - pthread_t update_index_thread; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/lock_manager.h b/src_dependent_variable_sized_reconnaissance_phases/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/scheduler.h b/src_dependent_variable_sized_reconnaissance_phases/scheduler/scheduler.h deleted file mode 100755 index d93465fa..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/scheduler.h +++ /dev/null @@ -1,19 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} - -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.h b/src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/sequencer/Makefile.inc b/src_dependent_variable_sized_reconnaissance_phases/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.cc b/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.cc deleted file mode 100755 index 8301fbec..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.cc +++ /dev/null @@ -1,387 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, ConnectionMultiplexer* multiplexer, - Client* client, Storage* storage) - : epoch_duration_(0.01), configuration_(conf), multiplexer_(multiplexer), - client_(client), storage_(storage), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - // Start Sequencer main loops running in background thread. - -message_queues = new AtomicQueue(); -restart_queues = new AtomicQueue(); -connection_ = multiplexer_->NewConnection("sequencer", &message_queues, &restart_queues); -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(2, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); - delete message_queues; -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -#ifdef PREFETCHING -double PrefetchAll(Storage* storage, TxnProto* txn) { - double max_wait_time = 0; - double wait_time = 0; - for (int i = 0; i < txn->read_set_size(); i++) { - storage->Prefetch(txn->read_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - storage->Prefetch(txn->read_write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } - for (int i = 0; i < txn->write_set_size(); i++) { - storage->Prefetch(txn->write_set(i), &wait_time); - max_wait_time = MAX(max_wait_time, wait_time); - } -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) - prefetch_cold[txn->txn_id() / SAMPLE_RATE] = max_wait_time; -#endif - return max_wait_time; -} -#endif - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - -#ifdef PREFETCHING - multimap fetching_txns; -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - std::cout << "Starting sequencer.\n" << std::flush; - -unordered_map pending_txns; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); - - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (txn_id_offset < MAX_BATCH_SIZE && batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - - bool got_message = message_queues->Pop(&message); - if(got_message == true) { - if (message.type() == MessageProto::REMOTE_INDEX_RESULT) { - - assert(pending_txns.count(message.txn_id()) > 0); - txn = pending_txns[message.txn_id()]; - pending_txns.erase(message.txn_id()); - if (txn->txn_type() == 3) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - } - } - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } else { - - if (rand() % 100 < DEPENDENT_PERCENT) { - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset, 1); - - MessageProto message; - message.set_type(MessageProto::REMOTE_INDEX_REQUEST); - message.set_destination_channel("remote_index"); - message.set_destination_node(configuration_->this_node_id); - message.set_index_number(txn->index_number()); - message.set_source_node(configuration_->this_node_id); - message.set_source_channel("sequencer"); - message.set_txn_id(IntToString(txn->txn_id())); - connection_->Send(message); - - pending_txns[IntToString(txn->txn_id())] = txn; - txn_id_offset++; - - } else { - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset, 0); - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } - } - } else if (txn_id_offset >= MAX_BATCH_SIZE && batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - - bool got_message = message_queues->Pop(&message); - if(got_message == true) { - if (message.type() == MessageProto::REMOTE_INDEX_RESULT) { - assert(pending_txns.count(message.txn_id()) > 0); - txn = pending_txns[message.txn_id()]; - pending_txns.erase(message.txn_id()); - if (txn->txn_type() == 3) { - for (int i = 0; i < message.keys_size(); i++) { - txn->add_read_write_set(message.keys(i)); - } - } - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } - } - } - - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - - // Set up batch messages for each system node. - map batches; - for (map::iterator it = configuration_->all_nodes.begin(); - it != configuration_->all_nodes.end(); ++it) { - batches[it->first].set_destination_channel("scheduler_"); - batches[it->first].set_destination_node(it->first); - batches[it->first].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif - - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - } while (!got_batch); -#endif -//assert(batch_message.data_size() > 0); - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); -#ifdef LATENCY_TEST - if (txn.txn_id() % SAMPLE_RATE == 0) - watched_txn = txn.txn_id(); -#endif - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); -assert(txn.writers_size() == 1); -assert(txn.readers_size() == 1); - - bytes txn_data; - txn.SerializeToString(&txn_data); - - // Compute union of 'readers' and 'writers' (store in 'readers'). - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - // Insert txn into appropriate batches. - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - batches[*it].add_data(txn_data); - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - -#ifdef LATENCY_TEST - if (watched_txn != -1) { - sequencer_send[watched_txn] = GetTime(); - watched_txn = -1; - } -#endif - - // Report output. - if (GetTime() > time + 1) { -#ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; -#endif - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.h b/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.h deleted file mode 100755 index 70acee26..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/sequencer/sequencer.h +++ /dev/null @@ -1,137 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include -#include - -#include "common/types.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "common/connection.h" - -//#define PAXOS -//#define PREFETCHING -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 160 - -#define SAMPLES 100000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST - -using std::set; -using std::string; -using std::queue; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -#define HOT 10000 - -//----------------------------------- -#define INDEX_NUMBER 10000 -#define DEPENDENT_PERCENT 100 -#define VOLATILITY 0 -extern unordered_map > index_; -//----------------------------------- - - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id, int dependent) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, ConnectionMultiplexer* multiplexer, Client* client, - Storage* storage); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - ConnectionMultiplexer* multiplexer_; - - // Client from which to get incoming txns. - Client* client_; - - // Pointer to this node's storage object, for prefetching. - Storage* storage_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; - - AtomicQueue* message_queues; - AtomicQueue* restart_queues; - -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/collapsed_versioned_storage_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/configuration_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/connection_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_lock_manager_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_scheduler_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/fetching_storage_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/microbenchmark_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/paxos_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/profile_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/sequencer_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/serial_scheduler_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/simple_storage_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/storage_manager_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/tpcc_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_dependent_variable_sized_reconnaissance_phases/tests/types_test.cc b/src_dependent_variable_sized_reconnaissance_phases/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_dependent_variable_sized_reconnaissance_phases/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_single_thread_vll/.cproject b/src_single_thread_vll/.cproject deleted file mode 100755 index 61ad6f76..00000000 --- a/src_single_thread_vll/.cproject +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src_single_thread_vll/.project b/src_single_thread_vll/.project deleted file mode 100755 index 590162ae..00000000 --- a/src_single_thread_vll/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - src - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - all - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/src_single_thread_vll/Makefile b/src_single_thread_vll/Makefile deleted file mode 100755 index e3e6e17d..00000000 --- a/src_single_thread_vll/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_single_thread_vll/Makefile.template b/src_single_thread_vll/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_single_thread_vll/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_single_thread_vll/applications/.DS_Store b/src_single_thread_vll/applications/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_single_thread_vll/applications/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/applications/Makefile.inc b/src_single_thread_vll/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_single_thread_vll/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/applications/application.h b/src_single_thread_vll/applications/application.h deleted file mode 100755 index 7b1c7d3b..00000000 --- a/src_single_thread_vll/applications/application.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config, int partition_id) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, - Configuration* conf, - int partition_id) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_single_thread_vll/applications/microbenchmark.cc b/src_single_thread_vll/applications/microbenchmark.cc deleted file mode 100755 index 003fd41a..00000000 --- a/src_single_thread_vll/applications/microbenchmark.cc +++ /dev/null @@ -1,155 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - txn->set_status(TxnProto::NEW); - - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { - assert(part1 != part2 || nparts == 1); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - txn->set_multipartition(true); - txn->set_status(TxnProto::NEW); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config, int partition_id) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - for (int i = 0; i < kRWSetSize; i++) { - ValueStore* val = storage->ReadObject(txn->read_write_set(i)); - - // The following is for "long" transactions. - int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - - val->value = IntToString(StringToInt(val->value) + 1); - val->exclusive_lock_number --; - } - - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, - Configuration* conf, - int partition_id) const { - for (int i = 0; i < nparts*kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == partition_id) { - storage->PutObject(IntToString(i), new ValueStore(IntToString(i), 0, 0)); - } - } -} - diff --git a/src_single_thread_vll/applications/microbenchmark.h b/src_single_thread_vll/applications/microbenchmark.h deleted file mode 100755 index 145bec12..00000000 --- a/src_single_thread_vll/applications/microbenchmark.h +++ /dev/null @@ -1,54 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL, int partition_id = 0) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - virtual void InitializeStorage(Storage* storage, Configuration* conf, int partition_id) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_single_thread_vll/applications/tpcc.cc b/src_single_thread_vll/applications/tpcc.cc deleted file mode 100755 index e8cdd133..00000000 --- a/src_single_thread_vll/applications/tpcc.cc +++ /dev/null @@ -1,1171 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. - - -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config, int partition_id) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_partition; - if (mp) { - do { - remote_partition = rand() % config->partitions_size; - } while (config->partitions_size > 1 && - remote_partition == partition_id); - - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - Value customer_value; - std::set items_used; - - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->partitions_size+ partition_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 10) + 5; - -int order_number; -if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; -} -else { - order_number = 0; - next_order_id_for_district[district_key] = 1; -} - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - remote_warehouse_id = (rand() % WAREHOUSES_PER_NODE) * - config->partitions_size + remote_partition; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - } - - // Determine if we should add it to read set to avoid duplicates - - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_major_partition(partition_id); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - - if (mp) - txn->set_multipartition(true); - - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->partitions_size + partition_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->partitions_size == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - - remote_warehouse_id = (rand() % WAREHOUSES_PER_NODE) * - config->partitions_size + remote_partition; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - - } - - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - break; - - // <--- Order-status transaction ---> - case ORDER_STATUS: -{ - -string customer_string; -string customer_latest_order; -string warehouse_string; -string district_string; -int customer_order_line_number; - -if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; -} - -pthread_mutex_lock(&mutex_); -customer_string = (*involed_customers)[rand() % involed_customers->size()]; - -pthread_mutex_unlock(&mutex_); - -customer_latest_order = latest_order_id_for_customer[customer_string]; - -warehouse_string = customer_string.substr(0,customer_string.find("d")); -district_string = customer_string.substr(0,customer_string.find("c")); -snprintf(customer_key, sizeof(customer_key), "%s", - customer_string.c_str()); -snprintf(warehouse_key, sizeof(warehouse_key), "%s", - warehouse_string.c_str()); -snprintf(district_key, sizeof(district_key), "%s", - district_string.c_str()); - - -customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s", - customer_latest_order.c_str()); - txn->add_read_set(order_key); - - char order_line_key[128]; -for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, i); - txn->add_read_set(order_line_key); -} - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - -} - case STOCK_LEVEL: -{ - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->partitions_size + partition_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - - int item = item_for_order_line[order_line_key]; - - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - -} - case DELIVERY : -{ - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->partitions_size + partition_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - - snprintf(district_key, sizeof(district_key), "%sd%d", - warehouse_key, i); - - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])) { - continue; - } else { - - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, oldest_order); - - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - txn->add_read_write_set(order_line_key); - } -string customer_string = customer_for_order[order_key]; - snprintf(customer_key, sizeof(customer_key), - "%s", customer_string.c_str()); - - txn->add_read_write_set(customer_key); - } - break; -} - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - // InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - - // First, we retrieve the warehouse from storage - Warehouse* warehouse = new Warehouse(); - ValueStore* warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(warehouse_value->value)); - warehouse_value->share_lock_number--; - - // Next, we retrieve the district - District* district = new District(); - ValueStore* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(district_value->value)); - - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(&(district_value->value))); - district_value->exclusive_lock_number--; - - // Retrieve the customer we are looking for - Customer* customer = new Customer(); - ValueStore* customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(customer_value->value)); - customer_value->share_lock_number--; - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - double system_time = tpcc_args->system_time(); - int major_partition = tpcc_args->major_partition(); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - char order_line_key[128]; - - snprintf(order_line_key, sizeof(order_line_key), "%s", - txn->write_set(i).c_str()); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_number(i); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - ValueStore* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(stock_value->value)); - stock_value->exclusive_lock_number--; - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(&(stock_value->value))); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - ValueStore* order_line_value_store = new ValueStore((*order_line_value), 0, 0); - storage->PutObject(order_line_key, order_line_value_store); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); -pthread_mutex_lock(&mutex_for_item); -if (storage->partition_id_ == major_partition) -item_for_order_line[order_line_key] = StringToInt(item_key); -pthread_mutex_unlock(&mutex_for_item); - - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), "%s", - txn->write_set(order_line_count).c_str()); - - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - ValueStore* new_order_value_store = new ValueStore((*new_order_value), 0, 0); - storage->PutObject(new_order_key, new_order_value_store); - - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%s", - txn->write_set(order_line_count + 1).c_str()); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - ValueStore* order_value_store = new ValueStore((*order_value), 0, 0); - storage->PutObject(order_key, order_value_store); - -int order_number = tpcc_args->order_number(); - -if (storage->partition_id_ == major_partition) { - -pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } -order_line_number[order_key] = order_line_count; -customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; -pthread_mutex_unlock(&mutex_); -} - - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - ValueStore* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || - secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - ValueStore* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(warehouse_value->value)); - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(&(warehouse_value->value))); - warehouse_value->exclusive_lock_number--; - - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - ValueStore* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(district_value->value)); - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(&(district_value->value))); - district_value->exclusive_lock_number--; - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(customer_value->value)); - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + - amount); - customer->set_payment_count(customer->payment_count() + 1); - customer_value->exclusive_lock_number--; - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(&(customer_value->value))); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value_data = new Value(); - ValueStore* history_value = new ValueStore((*history_value_data), 0, 0); - assert(history->SerializeToString(history_value_data)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - - return SUCCESS; -} - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - - Key warehouse_key = txn->read_set(0); - ValueStore* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(warehouse_value->value)); - warehouse_value->share_lock_number--; - District* district = new District(); - ValueStore* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(district_value->value)); - district_value->share_lock_number--; - ValueStore* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(customer_value->value)); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - customer_value->share_lock_number--; - Order* order = new Order(); - ValueStore* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(order_value->value)); - order_value->share_lock_number--; - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - ValueStore* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(order_line_value->value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - - order_line_value->share_lock_number--; - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - return SUCCESS; -} - - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - ValueStore* warehouse_value = storage->ReadObject(txn->read_set(0)); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(warehouse_value->value)); - warehouse_value->share_lock_number--; - - District* district = new District(); - ValueStore* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(district_value->value)); - district_value->share_lock_number--; - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - - OrderLine* order_line = new OrderLine(); - ValueStore* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(order_line_value->value)); - string item_key = order_line->item_id(); - order_line_value->share_lock_number--; - - Stock* stock = new Stock(); - ValueStore* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(stock_value->value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - stock_value->share_lock_number--; - - - delete order_line; - delete stock; - - } - - - delete warehouse; - delete district; - return SUCCESS; -} - - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - ValueStore* warehouse_value = storage->ReadObject(txn->read_set(0)); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(warehouse_value->value)); - warehouse_value->share_lock_number--; - -if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; -} - - -int delivery_district_number = txn->read_set_size() - 1; -int read_write_index = 0; -int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { -string district_string = txn->read_set(i); - District* district = new District(); - ValueStore* district_value = storage->ReadObject(district_string); - assert(district->ParseFromString(district_value->value)); - district_value->share_lock_number--; - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - ValueStore* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(order_value->value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - order_value->exclusive_lock_number--; - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - ValueStore* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(order_line_value->value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - order_line_value->exclusive_lock_number--; - - delete order_line; - } - - - Customer* customer = new Customer(); - ValueStore* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(customer_value->value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - customer_value->exclusive_lock_number--; - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; - -} - - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf, int partition_id) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->partitions_size); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); -if (conf->LookupPartition(warehouse_key) != partition_id) { - continue; -} - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(&warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == partition_id) { - storage->PutObject(warehouse_key, new ValueStore(warehouse_value, 0, 0)); - storage->PutObject(warehouse_key_ytd, new ValueStore(warehouse_value, 0, 0)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - // Next we initialize the object and serialize it - Value district_value; - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(&district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == partition_id) { - storage->PutObject(district_key, new ValueStore(district_value, 0, 0)); - storage->PutObject(district_key_ytd, new ValueStore(district_value, 0, 0)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value customer_value; - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(&customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == partition_id) - storage->PutObject(customer_key, new ValueStore(customer_value, 0, 0)); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(&stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == partition_id) - storage->PutObject(stock->id(), new ValueStore(stock_value, 0, 0)); - delete stock; - } - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } - - -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(0); - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_single_thread_vll/applications/tpcc.h b/src_single_thread_vll/applications/tpcc.h deleted file mode 100755 index c27c48fd..00000000 --- a/src_single_thread_vll/applications/tpcc.h +++ /dev/null @@ -1,126 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/utils.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 3 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - - - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER, - PAYMENT, - ORDER_STATUS, - DELIVERY, - STOCK_LEVEL, - }; - - -// TPCC() {} - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config, int partition_id) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf, int partition_id) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_single_thread_vll/backend/.DS_Store b/src_single_thread_vll/backend/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_single_thread_vll/backend/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/backend/Makefile.inc b/src_single_thread_vll/backend/Makefile.inc deleted file mode 100755 index bbd628f7..00000000 --- a/src_single_thread_vll/backend/Makefile.inc +++ /dev/null @@ -1,25 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/backend/collapsed_versioned_storage.cc b/src_single_thread_vll/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_single_thread_vll/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_single_thread_vll/backend/collapsed_versioned_storage.h b/src_single_thread_vll/backend/collapsed_versioned_storage.h deleted file mode 100755 index 249a40d6..00000000 --- a/src_single_thread_vll/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_single_thread_vll/backend/fetching_storage.cc b/src_single_thread_vll/backend/fetching_storage.cc deleted file mode 100755 index f5514372..00000000 --- a/src_single_thread_vll/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_single_thread_vll/backend/fetching_storage.h b/src_single_thread_vll/backend/fetching_storage.h deleted file mode 100755 index 273dc04c..00000000 --- a/src_single_thread_vll/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_single_thread_vll/backend/simple_storage.cc b/src_single_thread_vll/backend/simple_storage.cc deleted file mode 100755 index b1ba6437..00000000 --- a/src_single_thread_vll/backend/simple_storage.cc +++ /dev/null @@ -1,24 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -ValueStore* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, ValueStore* value, int64 txn_id) { - objects_[key] = value; - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} diff --git a/src_single_thread_vll/backend/simple_storage.h b/src_single_thread_vll/backend/simple_storage.h deleted file mode 100755 index 674ec4e3..00000000 --- a/src_single_thread_vll/backend/simple_storage.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include "common/utils.h" - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual ValueStore* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, ValueStore* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - - private: - unordered_map objects_; -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_single_thread_vll/backend/storage.h b/src_single_thread_vll/backend/storage.h deleted file mode 100755 index d8fdcece..00000000 --- a/src_single_thread_vll/backend/storage.h +++ /dev/null @@ -1,56 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -struct ValueStore { - ValueStore(Value v, int e, int s) : value(v), exclusive_lock_number(e), share_lock_number(s) {} - // The actual value - Value value; - int exclusive_lock_number; - int share_lock_number; -}; - - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual ValueStore* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, ValueStore* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_single_thread_vll/backend/storage_manager.cc b/src_single_thread_vll/backend/storage_manager.cc deleted file mode 100755 index 43b63fa0..00000000 --- a/src_single_thread_vll/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection,Storage* actual_storage, - TxnProto* txn, int partition_id, ValueStore* cache_storage[]) - : configuration_(config), connection_(connection),actual_storage_(actual_storage), - txn_(txn), partition_id_(partition_id){ - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == partition_id_) - reader = true; - } - - if (reader) { - message.set_type(MessageProto::READ_RESULT); -int j = 0; -for(int i = 0; i < txn->read_write_set_size(); i++) { - if (configuration_->LookupPartition(txn->read_write_set(i)) == - partition_id_) { - objects_[txn->read_write_set(i)] = cache_storage[j]; - message.add_keys(txn->read_write_set(i)); - message.add_values(cache_storage[j]->value); - j++; - } -} - -for(int i = 0; i < txn->read_set_size(); i++) { - if (configuration_->LookupPartition(txn->read_set(i)) == - partition_id_) { - objects_[txn->read_set(i)] = cache_storage[j]; - message.add_keys(txn->read_set(i)); - message.add_values(cache_storage[j]->value); - j++; - } -} - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != partition_id_) { - message.set_destination_node(txn->writers(i) / WorkersNumber); - string channel = IntToString(txn->writers(i)); - channel.append(IntToString(txn->txn_id())); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == partition_id_) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - ValueStore* val = new ValueStore(message.values(i), 1 , 1); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -ValueStore* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, ValueStore* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_single_thread_vll/backend/storage_manager.h b/src_single_thread_vll/backend/storage_manager.h deleted file mode 100755 index f33ff833..00000000 --- a/src_single_thread_vll/backend/storage_manager.h +++ /dev/null @@ -1,90 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" -#include "backend/storage.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, Storage* actual_storage, - TxnProto* txn, int partition_id,ValueStore* cache_storage[]); - - ~StorageManager(); - - ValueStore* ReadObject(const Key& key); - bool PutObject(const Key& key, ValueStore* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - int partition_id_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -// int repeated_reads; -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_single_thread_vll/backend/versioned_storage.h b/src_single_thread_vll/backend/versioned_storage.h deleted file mode 100755 index 24c6223b..00000000 --- a/src_single_thread_vll/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_single_thread_vll/common/.DS_Store b/src_single_thread_vll/common/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_single_thread_vll/common/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/common/Makefile.inc b/src_single_thread_vll/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_single_thread_vll/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/common/configuration.cc b/src_single_thread_vll/common/configuration.cc deleted file mode 100755 index 773afb1b..00000000 --- a/src_single_thread_vll/common/configuration.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include "common/configuration.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); - partitions_size = all_nodes.size() * WorkersNumber; -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(partitions_size); - else - return StringToInt(key) % static_cast(partitions_size); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - node->node_id, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - - } -} - diff --git a/src_single_thread_vll/common/configuration.h b/src_single_thread_vll/common/configuration.h deleted file mode 100755 index a0d3e41b..00000000 --- a/src_single_thread_vll/common/configuration.h +++ /dev/null @@ -1,98 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; -// The work thread number of one node -#define WorkersNumber 5 -#define ORDER_LINE_NUMBER 10 - - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - - -struct Node { - - int node_id; - int replica_id; - // Globally unique partition identifier. - int partition_id; - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - int partitions_size; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_single_thread_vll/common/configuration_test.conf b/src_single_thread_vll/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_single_thread_vll/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_single_thread_vll/common/configuration_test_one_node.conf b/src_single_thread_vll/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_single_thread_vll/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_single_thread_vll/common/connection.cc b/src_single_thread_vll/common/connection.cc deleted file mode 100755 index 7e727fea..00000000 --- a/src_single_thread_vll/common/connection.cc +++ /dev/null @@ -1,383 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if (new_connection_channel_->substr(0, 6) == "worker") { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - - if (message.type() == MessageProto::READ_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - if (message.destination_node() == multiplexer()->configuration()->this_node_id) { - Send(message); - } else { - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); - } -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_single_thread_vll/common/connection.h b/src_single_thread_vll/common/connection.h deleted file mode 100755 index 3e23300d..00000000 --- a/src_single_thread_vll/common/connection.h +++ /dev/null @@ -1,199 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - Configuration* configuration() {return configuration_;} - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_single_thread_vll/common/test_template.cc b/src_single_thread_vll/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_single_thread_vll/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_single_thread_vll/common/testing.h b/src_single_thread_vll/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_single_thread_vll/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_single_thread_vll/common/types.h b/src_single_thread_vll/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_single_thread_vll/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_single_thread_vll/common/utils.h b/src_single_thread_vll/common/utils.h deleted file mode 100755 index ca0630d5..00000000 --- a/src_single_thread_vll/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_single_thread_vll/common/zmq.hpp b/src_single_thread_vll/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_single_thread_vll/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_single_thread_vll/deployment/.DS_Store b/src_single_thread_vll/deployment/.DS_Store deleted file mode 100644 index 8df6bf6b..00000000 Binary files a/src_single_thread_vll/deployment/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/deployment/Makefile.inc b/src_single_thread_vll/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_single_thread_vll/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_single_thread_vll/deployment/addhosts.pl b/src_single_thread_vll/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_single_thread_vll/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_single_thread_vll/deployment/cluster.cc b/src_single_thread_vll/deployment/cluster.cc deleted file mode 100755 index 76b9b92b..00000000 --- a/src_single_thread_vll/deployment/cluster.cc +++ /dev/null @@ -1,379 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_single_thread_vll/deployment/main.cc b/src_single_thread_vll/deployment/main.cc deleted file mode 100755 index 19d233f8..00000000 --- a/src_single_thread_vll/deployment/main.cc +++ /dev/null @@ -1,183 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/simple_storage.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size() * WorkersNumber, HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->partitions_size > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int first = rand() % WorkersNumber + config_->this_node_id * WorkersNumber; - int other; - do { - other = rand() % config_->partitions_size; - } while (other == first); - *txn = microbenchmark.MicroTxnMP(txn_id, first, other); - } else { - // Single-partition txn. - int single_partition; - // srand(time(NULL)); - single_partition = rand() % WorkersNumber + config_->this_node_id * WorkersNumber; - *txn = microbenchmark.MicroTxnSP(txn_id, single_partition); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - int rand_local_partition = rand() % WorkersNumber + config_->this_node_id * WorkersNumber; - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_, rand_local_partition); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_, rand_local_partition); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_, rand_local_partition); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_, rand_local_partition); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_, rand_local_partition); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } -// bool useFetching = false; -// if (argc > 4 && argv[4][0] == 'f') -// useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif - // Create the storage of the partition - -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - - Storage* storage[WorkersNumber]; - for(int i = 0; i < WorkersNumber; i++) { - storage[i] = new SimpleStorage(); - if (argv[2][0] == 'm') { - Microbenchmark(config.partitions_size, HOT).InitializeStorage(storage[i], &config, config.this_node_id * WorkersNumber + i); - } else { - TPCC().InitializeStorage(storage[i], &config, config.this_node_id * WorkersNumber + i); - } - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client); - - // Initialize scheduler component and start sequencer thread running. - if (argv[2][0] == 'm') { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new Microbenchmark(config.partitions_size, HOT)); - } else { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new TPCC()); - } - - Spin(120); - return 0; -} - diff --git a/src_single_thread_vll/deployment/makeconf.pl b/src_single_thread_vll/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_single_thread_vll/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_single_thread_vll/deployment/portfile b/src_single_thread_vll/deployment/portfile deleted file mode 100755 index 35f317d7..00000000 --- a/src_single_thread_vll/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -54381 diff --git a/src_single_thread_vll/deployment/run.pbs b/src_single_thread_vll/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_single_thread_vll/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_single_thread_vll/deployment/test.conf b/src_single_thread_vll/deployment/test.conf deleted file mode 100755 index e3f09e31..00000000 --- a/src_single_thread_vll/deployment/test.conf +++ /dev/null @@ -1,11 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:16:128.36.232.37:50001 # cobra -node1=0:1:16:128.36.232.39:50002 # viper -node2=0:2:16:128.36.232.42:50003 # turtle -node3=0:3:16:128.36.232.50:50004 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_single_thread_vll/paxos/Makefile.inc b/src_single_thread_vll/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_single_thread_vll/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/paxos/paxos.cc b/src_single_thread_vll/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_single_thread_vll/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_single_thread_vll/paxos/paxos.h b/src_single_thread_vll/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_single_thread_vll/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_single_thread_vll/paxos/zookeeper.conf b/src_single_thread_vll/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_single_thread_vll/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_single_thread_vll/proto/.DS_Store b/src_single_thread_vll/proto/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_single_thread_vll/proto/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/proto/Makefile.inc b/src_single_thread_vll/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_single_thread_vll/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_single_thread_vll/proto/message.proto b/src_single_thread_vll/proto/message.proto deleted file mode 100755 index 83f7870c..00000000 --- a/src_single_thread_vll/proto/message.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; -} - diff --git a/src_single_thread_vll/proto/tpcc.proto b/src_single_thread_vll/proto/tpcc.proto deleted file mode 100755 index bc3143fb..00000000 --- a/src_single_thread_vll/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_single_thread_vll/proto/tpcc_args.proto b/src_single_thread_vll/proto/tpcc_args.proto deleted file mode 100755 index 77588384..00000000 --- a/src_single_thread_vll/proto/tpcc_args.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - optional int32 lastest_order_number = 41; - optional int32 threshold = 42; - optional int32 major_partition = 51; - optional int32 order_number = 52; -} diff --git a/src_single_thread_vll/proto/txn.proto b/src_single_thread_vll/proto/txn.proto deleted file mode 100755 index 49ce4723..00000000 --- a/src_single_thread_vll/proto/txn.proto +++ /dev/null @@ -1,58 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - BLOCKED = 4; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; -} - diff --git a/src_single_thread_vll/scheduler/.DS_Store b/src_single_thread_vll/scheduler/.DS_Store deleted file mode 100644 index 956a6bf7..00000000 Binary files a/src_single_thread_vll/scheduler/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/scheduler/Makefile.inc b/src_single_thread_vll/scheduler/Makefile.inc deleted file mode 100755 index 70613821..00000000 --- a/src_single_thread_vll/scheduler/Makefile.inc +++ /dev/null @@ -1,22 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/scheduler/deterministic_lock_manager.cc b/src_single_thread_vll/scheduler/deterministic_lock_manager.cc deleted file mode 100755 index 12cc9d59..00000000 --- a/src_single_thread_vll/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle write lock requests. - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) { -// // Only lock local keys. -// if (IsLocal(txn->write_set(i))) { -// deque* requests = lock_table_[Hash(txn->write_set(i))]; -// // Only need to request this if lock txn hasn't already requested it. -// if (requests->empty() || txn != requests->back().txn) { -// requests->push_back(LockRequest(WRITE, txn)); -// // Write lock request fails if there is any previous request at all. -// if (requests->size() > 1) -// not_acquired++; -// } -// } -// } - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* requests = lock_table_[Hash(txn->read_write_set(i))]; - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* requests = lock_table_[Hash(txn->read_set(i))]; - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* requests = lock_table_[Hash(key)]; - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); -// if (requests->size() == 0) -// lock_table_.erase(key); - } -} - diff --git a/src_single_thread_vll/scheduler/deterministic_lock_manager.h b/src_single_thread_vll/scheduler/deterministic_lock_manager.h deleted file mode 100755 index 7fae019c..00000000 --- a/src_single_thread_vll/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,76 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_single_thread_vll/scheduler/deterministic_scheduler.cc b/src_single_thread_vll/scheduler/deterministic_scheduler.cc deleted file mode 100755 index 258cecef..00000000 --- a/src_single_thread_vll/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,392 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include -#include "applications/tpcc.h" -#include "applications/application.h" -#include "common/utils.h" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using std::bitset; - -unordered_map batches[WorkersNumber]; - - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - ConnectionMultiplexer* multiplexer, - Storage* storage[], - const Application* application) - : configuration_(conf), multiplexer_(multiplexer){ -printf("Enter DeterministicScheduler\n"); - for (int i = 0; i < WorkersNumber; i++) - storage_[i] = storage[i]; - application_ = application; - for (int i = 0; i < WorkersNumber; i++) { - string channel_batch("scheduler"); - channel_batch.append(IntToString(i)); - batch_connections_[i] = multiplexer_->NewConnection(channel_batch); - } - - for (int i = 0; i < WorkersNumber; i++) { - message_queues[i] = new AtomicQueue(); - } - -cpu_set_t cpuset; - - for (int i = 0; i < WorkersNumber; i++) { - string channel_worker("worker"); - channel_worker.append(IntToString(i)); - thread_connections_[i] = multiplexer_->NewConnection(channel_worker, &message_queues[i]); - -pthread_attr_t attr; -pthread_attr_init(&attr); -CPU_ZERO(&cpuset); -if (i == 0 || i == 1) - CPU_SET(i, &cpuset); -else if (i == 2 || i == 3) - CPU_SET(i+2, &cpuset); -else - CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, reinterpret_cast( - new pair(configuration_->this_node_id * WorkersNumber + i, this))); - } -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -MessageProto* GetBatch(int partition_id, int batch_id, Connection* connection) { - if (batches[partition_id % WorkersNumber].count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[partition_id % WorkersNumber][batch_id]; - batches[partition_id % WorkersNumber].erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[partition_id % WorkersNumber][message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % 1000000; -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - - int partition_id = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - int thread = partition_id % WorkersNumber; - - MessageProto* batch_message = NULL; - MessageProto message; - int batch_offset = 0; - int batch_number = 0; - map TxnsQueue; - // The txn id of the current running txn - int txns = 0; - double time = GetTime(); - // double interal = GetTime(); - - int add = 0; - int idel = 0; - -printf("%d:thread assigned to CPU %d\n",partition_id, sched_getcpu()); - - // The active multi-partition txn - unordered_map active_txns; - Connection* this_connection = scheduler->thread_connections_[thread]; - Storage* this_storage = scheduler->storage_[thread]; - - ValueStore* cache_objects_[20]; - // For SCA - bitset<1000000> Dx; - bitset<1000000> Ds; - int blocked_txn = 0; -Spin(2); - while(true) { - - // First we check whether we can receive remote read - bool got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - assert(message.type() == MessageProto::READ_RESULT); - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - if(rand() % txn->writers_size() == 0) { - txns++; - } - delete manager; - this_connection->UnlinkChannel(message.destination_channel()); - - // Remove the txn from active_txns and wait_txns_ queue - active_txns.erase(message.destination_channel()); - - TxnsQueue.erase(txn->txn_id()); - - } - } else { - if (batch_message == NULL) { - batch_message = GetBatch(partition_id, batch_number, scheduler->batch_connections_[thread]); - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(partition_id, batch_number, scheduler->batch_connections_[thread]); - } else { - // Execute the front of wait_txns queue or execute the txn from the Batch - // Check whether we could execute the head of wait_txns. - // (wait_txns is not empty and the front transaction is a BLOCKED transation) - if (!TxnsQueue.empty() && TxnsQueue.begin()->second->status()== TxnProto::BLOCKED) { - blocked_txn--; - TxnProto* txn = TxnsQueue.begin()->second; - int j = 0; - - for(int i = 0; i < txn->read_write_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_write_set(i)) == partition_id) { - cache_objects_[j++] = this_storage->ReadObject(txn->read_write_set(i)); - } - } - - for(int i = 0; i < txn->read_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_set(i)) == partition_id) { - cache_objects_[j++] = this_storage->ReadObject(txn->read_set(i)); - } - } - - StorageManager* manager = new StorageManager(scheduler->configuration_, - this_connection,this_storage, - txn, partition_id,cache_objects_); - if (manager->ReadyToExecute()) { - scheduler->application_->Execute(txn, manager); - txns++; - TxnsQueue.erase(txn->txn_id()); - delete manager; - } else { - string channel = IntToString(partition_id); - channel.append(IntToString(txn->txn_id())); - this_connection->LinkChannel(channel); - active_txns[channel] = manager; - txn->set_status(TxnProto::ACTIVE); - } - } else if(blocked_txn <= 25){ - // If the number of BLOCKED transactions in the wait_txns queue is smaller than a Threshold, then execute the next transaction from batch - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - bool execute_now = true; - int j = 0; - - // Check whether can acquire all locks - for(int i = 0; i < txn->read_write_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_write_set(i)) == partition_id) { - cache_objects_[j] = this_storage->ReadObject(txn->read_write_set(i)); - if(cache_objects_[j]->exclusive_lock_number != 0 || cache_objects_[j]->share_lock_number != 0) { - execute_now = false; - } - cache_objects_[j]->exclusive_lock_number++; - j++; - } - } - - for (int i = 0; i < txn->read_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_set(i)) == partition_id) { - cache_objects_[j] = this_storage->ReadObject(txn->read_set(i)); - if(cache_objects_[j]->exclusive_lock_number != 0) { - execute_now = false; - } - cache_objects_[j]->share_lock_number++; - j++; - } - } - - // If can acquire all locks - if (execute_now == true) { - StorageManager* manager = new StorageManager(scheduler->configuration_, - this_connection,this_storage, - txn, partition_id,cache_objects_); - // If the txn is single-partition txn - if (manager->ReadyToExecute()) { - scheduler->application_->Execute(txn, manager); - txns ++; - delete manager; - } else { - // IF the txn is a multi-partition txn, add it to the active_txns and wait_txn. - string channel = IntToString(partition_id); - channel.append(IntToString(txn->txn_id())); - this_connection->LinkChannel(channel); - - active_txns[channel] = manager; - TxnsQueue.insert(std::pair(txn->txn_id(), txn)); - txn->set_status(TxnProto::ACTIVE); - - } - } else { - // Blocked txn, put it to wait_txns_ queue. - blocked_txn++; - TxnsQueue.insert(std::pair(txn->txn_id(), txn)); - txn->set_status(TxnProto::BLOCKED); - } - - - } else if(blocked_txn > 25) { - // Do SCA optimization - Dx.reset(); - Ds.reset(); - set erase_txns; - int hash_index; - - for(std::map::iterator it = TxnsQueue.begin(); it != TxnsQueue.end();it++) { - TxnProto* txn = it->second; - - // If the BLOCKED txn can be executed now, just execute it. - if(txn->status()== TxnProto::BLOCKED) { - bool execute_now = true; - - // Check for conflicts in WriteSet - for (int i = 0; i < txn->read_write_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_write_set(i)) == partition_id) { - hash_index = Hash(txn->read_write_set(i)); - if (Dx[hash_index] == 1 || Ds[hash_index] == 1) { - execute_now = false; - } - Dx[hash_index] = 1; - } - } - - // Check for conflicts in ReadSet - for (int i = 0; i < txn->read_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_set(i)) == partition_id) { - hash_index = Hash(txn->read_set(i)); - if (Dx[hash_index] == 1) { - execute_now = false; - } - Ds[hash_index] = 1; - } - } - - if(execute_now == true) { - add++; - blocked_txn--; - int j = 0; - - for(int i = 0; i < txn->read_write_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_write_set(i)) == partition_id) { - cache_objects_[j++] = this_storage->ReadObject(txn->read_write_set(i)); - } - } - - for(int i = 0; i < txn->read_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_set(i)) == partition_id) { - cache_objects_[j++] = this_storage->ReadObject(txn->read_set(i)); - } - } - - StorageManager* manager = new StorageManager(scheduler->configuration_, - this_connection,this_storage, - txn, partition_id,cache_objects_); - // If the txn is single-partition txn - if (manager->ReadyToExecute()) { - scheduler->application_->Execute(txn, manager); - txns ++; - delete manager; - erase_txns.insert(txn->txn_id()); - } else { - string channel = IntToString(partition_id); - channel.append(IntToString(txn->txn_id())); - this_connection->LinkChannel(channel); - - // Add it to the active_txns - active_txns[channel] = manager; - txn->set_status(TxnProto::ACTIVE); - } - } - } else { - // If the transaction is free, just mark the bit-array - for (int i = 0; i < txn->read_write_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_write_set(i)) == partition_id) { - hash_index = Hash(txn->read_write_set(i)); - Dx[hash_index] = 1; - } - } - for (int i = 0; i < txn->read_set_size(); i++) { - if (scheduler->configuration_->LookupPartition(txn->read_set(i)) == partition_id) { - hash_index = Hash(txn->read_set(i)); - Ds[hash_index] = 1; - } - } - } - } - - // Delete the single-partition blocked txn(multi-partition txn is still in wait_txn queue) - for(set::iterator it = erase_txns.begin(); it != erase_txns.end();it++) { - TxnsQueue.erase(*it); - } - } else { - idel++; - } - } - } - if(GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout<<"Partition:"<(txns)/total_time)<<" txns/sec."<<(int)TxnsQueue.size()<<" "<<(int)active_txns.size()<<" "<<(static_cast(add)/total_time)<<" "<<(static_cast(idel)/total_time)<<"\n"< - -#include - -#include "scheduler/scheduler.h" -#include -#include "common/connection.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/message.pb.h" - -using std::deque; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class ConnectionMultiplexer; -class DeterministicLockManager; -class Storage; -class TxnProto; -class Application; - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, ConnectionMultiplexer* multiplexer, - Storage* storage[], const Application* application); - virtual ~DeterministicScheduler(); - - private: - // Application currently being run. - const Application* application_; - - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[WorkersNumber]; - Connection* thread_connections_[WorkersNumber]; - - // Connection for receiving txn batches from sequencer. - Connection* batch_connections_[WorkersNumber]; - - ConnectionMultiplexer* multiplexer_; - - // Storage layer used in application execution. - Storage* storage_[WorkersNumber]; - - AtomicQueue* message_queues[WorkersNumber]; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_single_thread_vll/scheduler/lock_manager.h b/src_single_thread_vll/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_single_thread_vll/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_single_thread_vll/scheduler/scheduler.h b/src_single_thread_vll/scheduler/scheduler.h deleted file mode 100755 index 45557a1b..00000000 --- a/src_single_thread_vll/scheduler/scheduler.h +++ /dev/null @@ -1,18 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_single_thread_vll/scheduler/serial_scheduler.cc b/src_single_thread_vll/scheduler/serial_scheduler.cc deleted file mode 100755 index 1d2c9bfe..00000000 --- a/src_single_thread_vll/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_single_thread_vll/scheduler/serial_scheduler.h b/src_single_thread_vll/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_single_thread_vll/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_single_thread_vll/sequencer/.DS_Store b/src_single_thread_vll/sequencer/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_single_thread_vll/sequencer/.DS_Store and /dev/null differ diff --git a/src_single_thread_vll/sequencer/Makefile.inc b/src_single_thread_vll/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_single_thread_vll/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_single_thread_vll/sequencer/sequencer.cc b/src_single_thread_vll/sequencer/sequencer.cc deleted file mode 100755 index 2fd0b049..00000000 --- a/src_single_thread_vll/sequencer/sequencer.cc +++ /dev/null @@ -1,315 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "applications/tpcc.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST1 -double execute_begin[SAMPLES]; -double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - // Start Sequencer main loops running in background thread. - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - -CPU_ZERO(&cpuset); -CPU_SET(2, &cpuset); -//CPU_SET(3, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -CPU_ZERO(&cpuset); -//CPU_SET(2, &cpuset); -//CPU_SET(3, &cpuset); -CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { -std::cout << "~Sequencer() execute.\n" << std::flush; - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } -std::cout << "Starting sequencer, enter RunWriter.\n" << std::flush; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); - if(txn->txn_id() == -1) { - delete txn; - continue; - } -#ifdef LATENCY_TEST - if (txn->txn_id() % SAMPLE_RATE == 0) { - sequencer_recv[txn->txn_id() / SAMPLE_RATE] = - epoch_start - + epoch_duration_ * (static_cast(rand()) / RAND_MAX); - } -#endif -#ifdef LATENCY_TEST1 - if(txn->txn_id()%SAMPLE_RATE == 0) { - execute_begin[txn->txn_id()/SAMPLE_RATE] = GetTime(); - } -#endif - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } - } - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif -std::cout << "Starting sequencer, enter RunReader.\n" << std::flush; - // Set up batch messages for each system node. - map batches; - for (int i = 0; i < (int)configuration_->partitions_size; i++) { - string channel("scheduler"); - channel.append(IntToString(i%WorkersNumber)); - batches[i].set_destination_channel(channel); - batches[i].set_destination_node(i/WorkersNumber); - batches[i].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - } while (!got_batch); -#endif - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - -#ifdef LATENCY_TEST - if (txn.txn_id() % SAMPLE_RATE == 0) - watched_txn = txn.txn_id(); -#endif - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - - bytes txn_data; - txn.SerializeToString(&txn_data); - - // Compute union of 'readers' and 'writers' (store in 'readers'). - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - // Insert txn into appropriate batches. - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - batches[*it].add_data(txn_data); - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - -#ifdef LATENCY_TEST - if (watched_txn != -1) { - sequencer_send[watched_txn] = GetTime(); - watched_txn = -1; - } -#endif - // Report output. - if (GetTime() > time + 1) { -#ifdef VERBOSE_SEQUENCER - std::cout << "Submitted " << txn_count << " txns in " - << batch_count << " batches,\n" << std::flush; -#endif - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_single_thread_vll/sequencer/sequencer.h b/src_single_thread_vll/sequencer/sequencer.h deleted file mode 100755 index 95e56970..00000000 --- a/src_single_thread_vll/sequencer/sequencer.h +++ /dev/null @@ -1,116 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -//#define VERBOSE_SEQUENCER -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 500 - -#define SAMPLES 100000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST1 - -using std::set; -using std::string; -using std::queue; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST1 -extern double execute_begin[SAMPLES]; -extern double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_single_thread_vll/tests/collapsed_versioned_storage_test.cc b/src_single_thread_vll/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_single_thread_vll/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_single_thread_vll/tests/configuration_test.cc b/src_single_thread_vll/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_single_thread_vll/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_single_thread_vll/tests/connection_test.cc b/src_single_thread_vll/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_single_thread_vll/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_single_thread_vll/tests/deterministic_lock_manager_test.cc b/src_single_thread_vll/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_single_thread_vll/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_single_thread_vll/tests/deterministic_scheduler_test.cc b/src_single_thread_vll/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_single_thread_vll/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_single_thread_vll/tests/fetching_storage_test.cc b/src_single_thread_vll/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_single_thread_vll/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_single_thread_vll/tests/microbenchmark_test.cc b/src_single_thread_vll/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_single_thread_vll/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_single_thread_vll/tests/paxos_test.cc b/src_single_thread_vll/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_single_thread_vll/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_single_thread_vll/tests/profile_test.cc b/src_single_thread_vll/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_single_thread_vll/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_single_thread_vll/tests/sequencer_test.cc b/src_single_thread_vll/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_single_thread_vll/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_single_thread_vll/tests/serial_scheduler_test.cc b/src_single_thread_vll/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_single_thread_vll/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_single_thread_vll/tests/simple_storage_test.cc b/src_single_thread_vll/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_single_thread_vll/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_single_thread_vll/tests/storage_manager_test.cc b/src_single_thread_vll/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_single_thread_vll/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_single_thread_vll/tests/tpcc_test.cc b/src_single_thread_vll/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_single_thread_vll/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_single_thread_vll/tests/types_test.cc b/src_single_thread_vll/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_single_thread_vll/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_traditional_2pl_2pc/.cproject b/src_traditional_2pl_2pc/.cproject deleted file mode 100755 index 61ad6f76..00000000 --- a/src_traditional_2pl_2pc/.cproject +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src_traditional_2pl_2pc/.project b/src_traditional_2pl_2pc/.project deleted file mode 100755 index 590162ae..00000000 --- a/src_traditional_2pl_2pc/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - src - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - all - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/src_traditional_2pl_2pc/Makefile b/src_traditional_2pl_2pc/Makefile deleted file mode 100755 index fc13f190..00000000 --- a/src_traditional_2pl_2pc/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror -Wno-strict-aliasing -std=c++0x -fno-omit-frame-pointer - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_traditional_2pl_2pc/Makefile.template b/src_traditional_2pl_2pc/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_traditional_2pl_2pc/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_traditional_2pl_2pc/applications/Makefile.inc b/src_traditional_2pl_2pc/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_traditional_2pl_2pc/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/applications/application.h b/src_traditional_2pl_2pc/applications/application.h deleted file mode 100755 index 711284fa..00000000 --- a/src_traditional_2pl_2pc/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" -#include "backend/simple_storage.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_traditional_2pl_2pc/applications/microbenchmark.cc b/src_traditional_2pl_2pc/applications/microbenchmark.cc deleted file mode 100755 index 8eb8f9a0..00000000 --- a/src_traditional_2pl_2pc/applications/microbenchmark.cc +++ /dev/null @@ -1,201 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -int cmp(const void *a, const void *b) { - return *(int *)a - *(int *)b; -} - -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { -//if(part == 1) -//printf("Begin\n"); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - txn->set_multipartition(false); - txn->set_txn_node(part); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - -//if(part == 1) -//printf("End\n"); - return txn; -} -/** -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { -//if(part == 1) -//printf("Begin\n"); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - txn->set_multipartition(false); - txn->set_txn_node(part); - - // Add one hot key to read/write set. - int hotkey1 = part + nparts * (rand() % hot_records); - int hotkey2; - do { - hotkey2 = part + nparts * (rand() % hot_records); - } while(hotkey2 == hotkey1); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 2, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - -//if(part == 1) -//printf("End\n"); - return txn; -}**/ - -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2) { - assert(part1 != part2); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - txn->set_multipartition(true); - txn->set_txn_node(part1); - txn->add_txn_other_node(part2); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - GetRandomKeys(&keys, - kRWSetSize/2 - 1, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - - *val = IntToString(StringToInt(*val) + 1); - - int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - } - - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, Configuration* conf) const { - for (int i = 0; i < nparts * kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { - storage->PutObject(IntToString(i), new Value(IntToString(i))); - } - } -} - diff --git a/src_traditional_2pl_2pc/applications/microbenchmark.h b/src_traditional_2pl_2pc/applications/microbenchmark.h deleted file mode 100755 index 3c526cae..00000000 --- a/src_traditional_2pl_2pc/applications/microbenchmark.h +++ /dev/null @@ -1,55 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" -#include "backend/simple_storage.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part1); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_traditional_2pl_2pc/applications/tpcc.cc b/src_traditional_2pl_2pc/applications/tpcc.cc deleted file mode 100755 index 66a3b6b3..00000000 --- a/src_traditional_2pl_2pc/applications/tpcc.cc +++ /dev/null @@ -1,1103 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - txn->set_txn_node(config->this_node_id); - - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - txn->add_txn_other_node(remote_node); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s", warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s", district_string.c_str()); - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s", customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - pthread_mutex_lock(&mutex_); - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])) { - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - - txn->add_read_write_set(customer_key); - } - pthread_mutex_unlock(&mutex_); - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - - assert(order->ParseFromString(*order_value)); - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_traditional_2pl_2pc/applications/tpcc.h b/src_traditional_2pl_2pc/applications/tpcc.h deleted file mode 100755 index d6023c07..00000000 --- a/src_traditional_2pl_2pc/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 20 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_traditional_2pl_2pc/backend/Makefile.inc b/src_traditional_2pl_2pc/backend/Makefile.inc deleted file mode 100755 index bbd628f7..00000000 --- a/src_traditional_2pl_2pc/backend/Makefile.inc +++ /dev/null @@ -1,25 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.cc b/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.h b/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.h deleted file mode 100755 index 249a40d6..00000000 --- a/src_traditional_2pl_2pc/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc/backend/fetching_storage.cc b/src_traditional_2pl_2pc/backend/fetching_storage.cc deleted file mode 100755 index f5514372..00000000 --- a/src_traditional_2pl_2pc/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_traditional_2pl_2pc/backend/fetching_storage.h b/src_traditional_2pl_2pc/backend/fetching_storage.h deleted file mode 100755 index 273dc04c..00000000 --- a/src_traditional_2pl_2pc/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_traditional_2pl_2pc/backend/simple_storage.h b/src_traditional_2pl_2pc/backend/simple_storage.h deleted file mode 100755 index f15f69e9..00000000 --- a/src_traditional_2pl_2pc/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc/backend/storage.h b/src_traditional_2pl_2pc/backend/storage.h deleted file mode 100755 index a793360a..00000000 --- a/src_traditional_2pl_2pc/backend/storage.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc/backend/storage_manager.cc b/src_traditional_2pl_2pc/backend/storage_manager.cc deleted file mode 100755 index 79da935e..00000000 --- a/src_traditional_2pl_2pc/backend/storage_manager.cc +++ /dev/null @@ -1,217 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn, int partition_id) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn), partition_id_(partition_id) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. -received_prepared_reply_number = 0; -received_prepared_message = false; - -received_commit_reply_number = 0; -received_commit_message = false; - - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == partition_id_) - reader = true; - } - - if (reader) { - // message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - partition_id_) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == partition_id_) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != partition_id_) { - message.set_destination_node(txn->writers(i)); - string channel = IntToString(txn->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == partition_id_) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} -// -----------FOR 2PC prepard phase------------------ -bool StorageManager::ReceivedAllPreparedReplyMessages() { - return received_prepared_reply_number == txn_->writers_size() - 1; -} - -void StorageManager::BroadcastPreparedMessages() { - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn_->writers_size(); i++) { - if (txn_->writers(i) != partition_id_) { - MessageProto message; - message.set_type(MessageProto::PREPARED); - message.set_destination_node(txn_->writers(i)); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } -} - -void StorageManager::HandlePreparedReplyMessages(const MessageProto& message) { - assert(message.type() == MessageProto::PREPARED_REPLY); - received_prepared_reply_number ++; -} - -bool StorageManager::ReceivedPreparedMessages() { - return received_prepared_message == true; -} - -void StorageManager::HandlePreparedMessages(const MessageProto& message) { - assert(message.type() == MessageProto::PREPARED); - received_prepared_message = true; -} - -void StorageManager::SendPreparedReplyMessages() { - MessageProto message; - message.set_type(MessageProto::PREPARED_REPLY); - // Broadcast local reads to (other) writers. - - message.set_destination_node(txn_->txn_node()); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); -} -// -----------FOR 2PC prepard phase------------------ - - -// -----------FOR 2PC commit phase------------------ -bool StorageManager::ReceivedAllCommitReplyMessages() { - return received_commit_reply_number == txn_->writers_size() - 1; -} - -void StorageManager::BroadcastCommitMessages() { - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn_->writers_size(); i++) { - if (txn_->writers(i) != partition_id_) { - MessageProto message; - message.set_type(MessageProto::COMMIT); - message.set_destination_node(txn_->writers(i)); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } -} - -void StorageManager::HandleCommitReplyMessages(const MessageProto& message) { - assert(message.type() == MessageProto::COMMIT_REPLY); - received_commit_reply_number ++; -} - -bool StorageManager::ReceivedCommitMessages() { - return received_commit_message == true; -} - -void StorageManager::HandleCommitMessages(const MessageProto& message) { - assert(message.type() == MessageProto::COMMIT); - received_commit_message = true; -} - -void StorageManager::SendCommitReplyMessages() { - MessageProto message; - message.set_type(MessageProto::COMMIT_REPLY); - // Broadcast local reads to (other) writers. - - message.set_destination_node(txn_->txn_node()); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); -} -// -----------FOR 2PC commit phase------------------ - - - - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_traditional_2pl_2pc/backend/storage_manager.h b/src_traditional_2pl_2pc/backend/storage_manager.h deleted file mode 100755 index 9e748498..00000000 --- a/src_traditional_2pl_2pc/backend/storage_manager.h +++ /dev/null @@ -1,112 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" -#include "backend/storage.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn, int partition_id); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - bool ReceivedAllPreparedReplyMessages(); - void BroadcastPreparedMessages(); - void HandlePreparedReplyMessages(const MessageProto& message); - bool ReceivedPreparedMessages(); - void HandlePreparedMessages(const MessageProto& message); - void SendPreparedReplyMessages(); - - bool ReceivedAllCommitReplyMessages(); - void BroadcastCommitMessages(); - void HandleCommitReplyMessages(const MessageProto& message); - bool ReceivedCommitMessages(); - void HandleCommitMessages(const MessageProto& message); - void SendCommitReplyMessages(); - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - int partition_id_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - - int received_prepared_reply_number; - - bool received_prepared_message; - - int received_commit_reply_number; - - bool received_commit_message; - - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_traditional_2pl_2pc/backend/versioned_storage.h b/src_traditional_2pl_2pc/backend/versioned_storage.h deleted file mode 100755 index 24c6223b..00000000 --- a/src_traditional_2pl_2pc/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc/common/Makefile.inc b/src_traditional_2pl_2pc/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_traditional_2pl_2pc/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/common/configuration.cc b/src_traditional_2pl_2pc/common/configuration.cc deleted file mode 100755 index aede7dab..00000000 --- a/src_traditional_2pl_2pc/common/configuration.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - node->node_id, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - - } -} - diff --git a/src_traditional_2pl_2pc/common/configuration.h b/src_traditional_2pl_2pc/common/configuration.h deleted file mode 100755 index 6d1d256f..00000000 --- a/src_traditional_2pl_2pc/common/configuration.h +++ /dev/null @@ -1,101 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; -// The work thread number of one node -#define WorkersNumber 5 -#define SPINTIME 0.000001 - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - - -struct Node { - - int node_id; - int replica_id; - // Globally unique partition identifier. - int partition_id; - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - // Tracks the set of current active partitions in the system. -// map all_partitions; - // partition number of this node -// vector partitions_of_this_node; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_traditional_2pl_2pc/common/configuration_test.conf b/src_traditional_2pl_2pc/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_traditional_2pl_2pc/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_traditional_2pl_2pc/common/configuration_test_one_node.conf b/src_traditional_2pl_2pc/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_traditional_2pl_2pc/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_traditional_2pl_2pc/common/connection.cc b/src_traditional_2pl_2pc/common/connection.cc deleted file mode 100755 index 06515275..00000000 --- a/src_traditional_2pl_2pc/common/connection.cc +++ /dev/null @@ -1,401 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" -#include "sequencer/sequencer.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - - send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } - -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - if (new_connection_channel_->substr(0, 6) == "worker") { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - if (doing_deadlocks.count(message.channel_request()) > 0) { - MessageProto message1; - message1.set_type(MessageProto::TXN_ABORT); - message1.set_destination_channel(message.channel_request()); - message1.set_destination_node(configuration_->this_node_id); - Send(message1); - } - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - if ((message.destination_node() == configuration_->this_node_id) && (message.type() == MessageProto::TXN_ABORT)) { - if (undelivered_messages_.count(message.destination_channel()) > 0) { - undelivered_messages_.erase(message.destination_channel()); - remote_result_["force_abort_unactive_txn"]->Push(message); - doing_deadlocks[message.destination_channel()] = 1; - } else { - if (remote_result_.count(message.destination_channel()) == 0) { - remote_result_["force_abort_unactive_txn"]->Push(message); - doing_deadlocks[message.destination_channel()] = 1; - } else { - remote_result_[message.destination_channel()]->Push(message); - } - } - return ; - } - - if ((message.destination_node() == configuration_->this_node_id) && (message.type() == MessageProto::READ_RESULT || message.type() == MessageProto::PREPARED || - message.type() == MessageProto::PREPARED_REPLY || message.type() == MessageProto::COMMIT || - message.type() == MessageProto::COMMIT_REPLY || message.type() == MessageProto::WAIT_FOR_GRAPH)) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_traditional_2pl_2pc/common/connection.h b/src_traditional_2pl_2pc/common/connection.h deleted file mode 100755 index f8d52166..00000000 --- a/src_traditional_2pl_2pc/common/connection.h +++ /dev/null @@ -1,198 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_traditional_2pl_2pc/common/test_template.cc b/src_traditional_2pl_2pc/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_traditional_2pl_2pc/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_traditional_2pl_2pc/common/testing.h b/src_traditional_2pl_2pc/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_traditional_2pl_2pc/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_traditional_2pl_2pc/common/types.h b/src_traditional_2pl_2pc/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_traditional_2pl_2pc/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_traditional_2pl_2pc/common/utils.h b/src_traditional_2pl_2pc/common/utils.h deleted file mode 100755 index ca0630d5..00000000 --- a/src_traditional_2pl_2pc/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_traditional_2pl_2pc/common/zmq.hpp b/src_traditional_2pl_2pc/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_traditional_2pl_2pc/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_traditional_2pl_2pc/deployment/Makefile.inc b/src_traditional_2pl_2pc/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_traditional_2pl_2pc/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_traditional_2pl_2pc/deployment/addhosts.pl b/src_traditional_2pl_2pc/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_traditional_2pl_2pc/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_traditional_2pl_2pc/deployment/cluster.cc b/src_traditional_2pl_2pc/deployment/cluster.cc deleted file mode 100755 index 3c0e30d3..00000000 --- a/src_traditional_2pl_2pc/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_traditional_2pl_2pc/deployment/main.cc b/src_traditional_2pl_2pc/deployment/main.cc deleted file mode 100755 index 27d80c57..00000000 --- a/src_traditional_2pl_2pc/deployment/main.cc +++ /dev/null @@ -1,172 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/simple_storage.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -unordered_map doing_deadlocks; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other1; - do { - other1 = rand() % config_->all_nodes.size(); - } while (other1 == config_->this_node_id); - - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other1); - } else { - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - - pthread_mutex_init(&mutex_, NULL); - pthread_mutex_init(&mutex_for_item, NULL); - involed_customers = new vector; -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif - // Create the storage of the partition - Storage* storage = new SimpleStorage(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client); -storage->Initmutex(); - // Initialize scheduler component and start sequencer thread running. - if (argv[2][0] == 'm') { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new TPCC()); - } - - Spin(120); - return 0; -} - diff --git a/src_traditional_2pl_2pc/deployment/makeconf.pl b/src_traditional_2pl_2pc/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_traditional_2pl_2pc/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_traditional_2pl_2pc/deployment/portfile b/src_traditional_2pl_2pc/deployment/portfile deleted file mode 100755 index 57d37c4c..00000000 --- a/src_traditional_2pl_2pc/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -53527 diff --git a/src_traditional_2pl_2pc/deployment/run.pbs b/src_traditional_2pl_2pc/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_traditional_2pl_2pc/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_traditional_2pl_2pc/deployment/test.conf b/src_traditional_2pl_2pc/deployment/test.conf deleted file mode 100755 index 108bea6e..00000000 --- a/src_traditional_2pl_2pc/deployment/test.conf +++ /dev/null @@ -1,339 +0,0 @@ -# Node=:::: - - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:0:12:128.36.232.12:51261 #rhino -#node2=0:0:12:128.36.232.15:51262 # gator -#node3=0:0:12:128.36.232.20:51261 #macaw -#node4=0:0:12:128.36.232.19:51262 # peacock -#node5=0:0:12:128.36.232.34:51261 # perch -#node6=0:0:12:128.36.232.14:51262 # jaguar -#node7=0:0:12:128.36.232.16:51261 # hippo -#node8=0:0:12:128.36.232.39:51261 #viper -#node9=0:0:12:128.36.232.7:51261 #hornet -#node10=0:0:12:128.36.232.8:51261 #cricket -#node11=0:0:12:128.36.232.28:51261 #termite -#node12=0:0:12:128.36.232.5:51261 #cicada -#node13=0:0:12:128.36.232.6:51261 #bumblebee -#node14=0:0:12:128.36.232.44:51261 #frog -#node15=0:0:12:128.36.232.50:51261 #tick -#node16=0:0:12:128.36.232.42:51261 #turtle -#node17=0:0:12:128.36.232.35:51261 #aphid -#node18=0:0:12:128.36.232.41:51261 #chameleon -#node19=0:0:12:128.36.232.38:51261 #rattlesnake -#node20=0:0:12:128.36.232.45:51261 #scorpion -#node21=0:0:12:128.36.232.29:51261 #cardinal -#node22=0:0:12:128.36.232.24:51261 #swan -#node23=0:0:12:128.36.232.27:51261 #monkey - - - -#node2=0:8:16:128.36.232.16:51471 # lion -#node2=0:9:16:128.36.232.16:50005 -#node2=0:10:16:128.36.232.16:50005 -#node2=0:11:16:128.36.232.16:50005 -#node3=0:12:16:128.36.232.20:51472 -#node3=0:13:16:128.36.232.20:51473 -#node3=0:14:16:128.36.232.20:51473 -#node3=0:15:16:128.36.232.20:51473 - - - - - -#node0=0:0:16:128.36.232.9:51262 # lion -#node0=0:1:16:128.36.232.9:50005 -#node0=0:2:16:128.36.232.9:50005 -#node0=0:3:16:128.36.232.9:50005 -#node1=0:4:16:128.36.232.14:51471 # jaguar -#node1=0:5:16:128.36.232.14:50005 -#node1=0:6:16:128.36.232.14:50005 -#node1=0:7:16:128.36.232.14:50005 -#node2=0:8:16:128.36.232.15:51471 # gator -#node2=0:9:16:128.36.232.15:50005 -#node2=0:10:16:128.36.232.15:50005 -#node2=0:11:16:128.36.232.15:50005 -#node3=0:12:16:128.36.232.18:51471 # grizzly -#node3=0:13:16:128.36.232.18:50005 -#node3=0:14:16:128.36.232.18:50005 -#node3=0:15:16:128.36.232.18:50005 -#node4=0:16:16:128.36.232.16:51471 # hippo -#node4=0:17:16:128.36.232.16:50005 -#node4=0:18:16:128.36.232.16:50005 -#node4=0:19:16:128.36.232.16:50005 -#node5=0:20:16:128.36.232.34:51471 # perch -#node5=0:21:16:128.36.232.34:50005 -#node5=0:22:16:128.36.232.34:50005 -#node5=0:23:16:128.36.232.34:50005 -#node6=0:24:16:128.36.232.37:51471 # cobra -#node6=0:25:16:128.36.232.37:50005 -#node6=0:26:16:128.36.232.37:50005 -#node6=0:27:16:128.36.232.37:50005 -#node7=0:28:16:128.36.232.12:51471 # rhino -#node7=0:29:16:128.36.232.12:50005 -#node7=0:30:16:128.36.232.12:50005 -#node7=0:31:16:128.36.232.12:50005 -#node8=0:32:16:128.36.232.8:50005 # cricket -#node8=0:33:16:128.36.232.8:50005 # cricket -#node8=0:34:16:128.36.232.8:50005 # cricket -#node8=0:35:16:128.36.232.8:50005 # cricket -#node9=0:36:16:128.36.232.43:51471 # newt -#node9=0:37:16:128.36.232.43:50005 -#node9=0:38:16:128.36.232.43:50005 -#node9=0:39:16:128.36.232.43:50005 -#node10=0:40:16:128.36.232.45:51471 # scorpion -#node10=0:41:16:128.36.232.45:50005 -#node10=0:42:16:128.36.232.45:50005 -#node10=0:43:16:128.36.232.45:50005 -#node11=0:44:16:128.36.232.29:51471 # cardinal -#node11=0:45:16:128.36.232.29:50005 -#node11=0:46:16:128.36.232.29:50005 -#node11=0:47:16:128.36.232.29:50005 -#node12=0:48:16:128.36.232.27:51471 # monkey -#node12=0:49:16:128.36.232.27:50005 -#node12=0:50:16:128.36.232.27:50005 -#node12=0:51:16:128.36.232.27:50005 -#node13=0:52:16:128.36.232.39:51471 # viper -#node13=0:53:16:128.36.232.39:50005 -#node13=0:54:16:128.36.232.39:50005 -#node13=0:55:16:128.36.232.39:50005 -#node14=0:56:16:128.36.232.7:51471 # hornet -#node14=0:57:16:128.36.232.7:50005 -#node14=0:58:16:128.36.232.7:50005 -#node14=0:59:16:128.36.232.7:50005 -#node15=0:60:16:128.36.232.28:50005 # termite -#node15=0:61:16:128.36.232.28:50005 # termite -#node15=0:62:16:128.36.232.28:50005 # termite -#node15=0:63:16:128.36.232.28:50005 # termite -#node16=0:64:16:128.36.232.50:50005 # tick -#node16=0:65:16:128.36.232.50:50005 # tick -#node16=0:66:16:128.36.232.50:50005 # tick -#node16=0:67:16:128.36.232.50:50005 # tick -#node17=0:68:16:128.36.232.20:51471 # macaw -#node17=0:69:16:128.36.232.20:50005 -#node17=0:70:16:128.36.232.20:50005 -#node17=0:71:16:128.36.232.20:50005 -#node18=0:72:16:128.36.232.44:50004 # frog -#node18=0:73:16:128.36.232.44:50004 # frog -#node18=0:74:16:128.36.232.44:50004 # frog -#node18=0:75:16:128.36.232.44:50004 # frog -#node19=0:76:16:128.36.232.42:50003 # turtle -#node19=0:77:16:128.36.232.42:50003 # turtle -#node19=0:78:16:128.36.232.42:50003 # turtle -#node19=0:79:16:128.36.232.42:50003 # turtle - - -#node0=0:0:16:128.36.232.16:51262 # hippo -#node0=0:1:16:128.36.232.16:50005 -#node0=0:2:16:128.36.232.16:50005 -#node0=0:3:16:128.36.232.16:50005 -#node1=0:4:16:128.36.232.29:51471 # cardinal -#node1=0:5:16:128.36.232.29:50005 -#node1=0:6:16:128.36.232.29:50005 -#node1=0:7:16:128.36.232.29:50005 -#node2=0:8:16:128.36.232.6:51471 # bumblebee -#node2=0:9:16:128.36.232.6:50005 -#node2=0:10:16:128.36.232.6:50005 -#node2=0:11:16:128.36.232.6:50005 -#node3=0:12:16:128.36.232.18:51471 # grizzly -#node3=0:13:16:128.36.232.18:50005 -#node3=0:14:16:128.36.232.18:50005 -#node3=0:15:16:128.36.232.18:50005 -#node4=0:16:16:128.36.232.15:51471 # gator -#node4=0:17:16:128.36.232.15:50005 -#node4=0:18:16:128.36.232.15:50005 -#node4=0:19:16:128.36.232.15:50005 -#node5=0:20:16:128.36.232.38:51471 # rattlesnake -#node5=0:21:16:128.36.232.38:50005 -#node5=0:22:16:128.36.232.38:50005 -#node5=0:23:16:128.36.232.38:50005 -#node6=0:24:16:128.36.232.28:51471 # termite -#node6=0:25:16:128.36.232.28:50005 -#node6=0:26:16:128.36.232.28:50005 -#node6=0:27:16:128.36.232.28:50005 -#node7=0:28:16:128.36.232.34:51471 # perch -#node7=0:29:16:128.36.232.34:50005 -#node7=0:30:16:128.36.232.34:50005 -#node7=0:31:16:128.36.232.34:50005 -#node8=0:32:16:128.36.232.20:51471 # macaw -#node8=0:33:16:128.36.232.20:50005 -#node8=0:34:16:128.36.232.20:50005 -#node8=0:35:16:128.36.232.20:50005 -#node9=0:36:16:128.36.232.14:51471 # jaguar -#node9=0:37:16:128.36.232.14:50005 -#node9=0:38:16:128.36.232.14:50005 -#node9=0:39:16:128.36.232.14:50005 -#node10=0:40:16:128.36.232.7:51471 # hornet -#node10=0:41:16:128.36.232.7:50005 -#node10=0:42:16:128.36.232.7:50005 -#node10=0:43:16:128.36.232.7:50005 -#node11=0:44:16:128.36.232.27:51471 # monkey -#node11=0:45:16:128.36.232.27:50005 -#node11=0:46:16:128.36.232.27:50005 -#node11=0:47:16:128.36.232.27:50005 -#node12=0:48:16:128.36.232.43:51471 # newt -#node12=0:49:16:128.36.232.43:50005 -#node12=0:50:16:128.36.232.43:50005 -#node12=0:51:16:128.36.232.43:50005 -#node13=0:52:16:128.36.232.35:51471 # aphid -#node13=0:53:16:128.36.232.35:50005 -#node13=0:54:16:128.36.232.35:50005 -#node13=0:55:16:128.36.232.35:50005 -#node14=0:56:16:128.36.232.37:51471 # cobra -#node14=0:57:16:128.36.232.37:50005 -#node14=0:58:16:128.36.232.37:50005 -#node14=0:59:16:128.36.232.37:50005 -#node15=0:60:16:128.36.232.45:51471 # scorpion -#node15=0:61:16:128.36.232.45:50005 -#node15=0:62:16:128.36.232.45:50005 -#node15=0:63:16:128.36.232.45:50005 - - -#node0=0:0:16:128.36.232.7:51471 # hornet -#node0=0:1:16:128.36.232.7:50005 -#node0=0:2:16:128.36.232.7:50005 -#node0=0:3:16:128.36.232.7:50005 -#node1=0:4:16:128.36.232.27:51471 #monkey -#node1=0:5:16:128.36.232.27:50005 -#node1=0:6:16:128.36.232.27:50005 -#node1=0:7:16:128.36.232.27:50005 -#node2=0:8:16:128.36.232.8:51471 #cricket -#node2=0:9:16:128.36.232.8:50005 -#node2=0:10:16:128.36.232.8:50005 -#node2=0:11:16:128.36.232.8:50005 -#node3=0:12:16:128.36.232.15:51471 # gator -#node3=0:13:16:128.36.232.15:50005 -#node3=0:14:16:128.36.232.15:50005 -#node3=0:15:16:128.36.232.15:50005 -#node4=0:16:16:128.36.232.6:51471 # bumblebee -#node4=0:17:16:128.36.232.6:50005 -#node4=0:18:16:128.36.232.6:50005 -#node4=0:19:16:128.36.232.6:50005 -#node5=0:20:16:128.36.232.41:51471 # chameleon -#node5=0:21:16:128.36.232.41:50005 -#node5=0:22:16:128.36.232.41:50005 -#node5=0:23:16:128.36.232.41:50005 -#node6=0:24:16:128.36.232.49:51471 # ladybug -#node6=0:25:16:128.36.232.49:50005 -#node6=0:26:16:128.36.232.49:50005 -#node6=0:27:16:128.36.232.49:50005 -#node7=0:28:16:128.36.232.43:51471 # newt -#node7=0:29:16:128.36.232.43:50005 -#node7=0:30:16:128.36.232.43:50005 -#node7=0:31:16:128.36.232.43:50005 -#node8=0:32:16:128.36.232.39:51471 # viper -#node8=0:33:16:128.36.232.39:50005 -#node8=0:34:16:128.36.232.39:50005 -#node8=0:35:16:128.36.232.39:50005 -#node9=0:36:16:128.36.232.5:51471 # cicada -#node9=0:37:16:128.36.232.5:50005 -#node9=0:38:16:128.36.232.5:50005 -#node9=0:39:16:128.36.232.5:50005 -#node10=0:40:16:128.36.232.38:51471 # rattlesnake -#node10=0:41:16:128.36.232.38:50005 -#node10=0:42:16:128.36.232.38:50005 -#node10=0:43:16:128.36.232.38:50005 -#node11=0:44:16:128.36.232.29:51471 # cardinal -#node11=0:45:16:128.36.232.29:50005 -#node11=0:46:16:128.36.232.29:50005 -#node11=0:47:16:128.36.232.29:50005 -#node12=0:48:16:128.36.232.16:51471 # hippo -#node12=0:49:16:128.36.232.16:50005 -#node12=0:50:16:128.36.232.16:50005 -#node12=0:51:16:128.36.232.16:50005 -#node13=0:52:16:128.36.232.12:51471 # rhino -#node13=0:53:16:128.36.232.12:50005 -#node13=0:54:16:128.36.232.12:50005 -#node13=0:55:16:128.36.232.12:50005 -#node14=0:56:16:128.36.232.14:51471 # jaguar -#node14=0:57:16:128.36.232.14:50005 -#node14=0:58:16:128.36.232.14:50005 -#node14=0:59:16:128.36.232.14:50005 -#node15=0:60:16:128.36.232.20:51471 # macaw -#node15=0:61:16:128.36.232.20:50005 -#node15=0:62:16:128.36.232.20:50005 -#node15=0:63:16:128.36.232.20:50005 - -#node0=0:0:16:128.36.232.8:50005 # cricket -#node0=0:1:16:128.36.232.8:50005 # cricket -#node0=0:2:16:128.36.232.8:50005 # cricket -#node0=0:3:16:128.36.232.8:50005 # cricket -#node1=0:4:16:128.36.232.6:50005 # bumblebee -#node1=0:5:16:128.36.232.6:50005 # bumblebee -#node1=0:6:16:128.36.232.6:50005 # bumblebee -#node1=0:7:16:128.36.232.6:50005 # bumblebee -#node2=0:8:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:9:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:10:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:11:16:128.36.232.38:50005 # lion//rattlesnake -#node3=0:12:16:128.36.232.34:50001 # perch -#node3=0:13:16:128.36.232.34:50001 # perch -#node3=0:14:16:128.36.232.34:50001 # perch -#node3=0:15:16:128.36.232.34:50001 # perch -#node4=0:16:16:128.36.232.39:50002 # viper -#node4=0:17:16:128.36.232.39:50002 # viper -#node4=0:18:16:128.36.232.39:50002 # viper -#node4=0:19:16:128.36.232.39:50002 # viper -#node5=0:20:16:128.36.232.42:50003 # turtle -#node5=0:21:16:128.36.232.42:50003 # turtle -#node5=0:22:16:128.36.232.42:50003 # turtle -#node5=0:23:16:128.36.232.42:50003 # turtle -#node6=0:24:16:128.36.232.44:50004 # frog -#node6=0:25:16:128.36.232.44:50004 # frog -#node6=0:26:16:128.36.232.44:50004 # frog -#node6=0:27:16:128.36.232.44:50004 # frog -#node7=0:28:16:128.36.232.41:50005 # chameleon -#node7=0:29:16:128.36.232.41:50005 # chameleon -#node7=0:30:16:128.36.232.41:50005 # chameleon -#node7=0:31:16:128.36.232.41:50005 # chameleon -#node8=0:32:16:128.36.232.45:50005 # scorpion -#node8=0:33:16:128.36.232.45:50005 # scorpion -#node8=0:34:16:128.36.232.45:50005 # scorpion -#node8=0:35:16:128.36.232.45:50005 # scorpion -#node9=0:36:16:128.36.232.49:50005 # ladybug -#node9=0:37:16:128.36.232.49:50005 # ladybug -#node9=0:38:16:128.36.232.49:50005 # ladybug -#node9=0:39:16:128.36.232.49:50005 # ladybug -#node10=0:40:16:128.36.232.7:50005 # hornet -#node10=0:41:16:128.36.232.7:50005 # hornet -#node10=0:42:16:128.36.232.7:50005 # hornet -#node10=0:43:16:128.36.232.7:50005 # hornet -#node11=0:44:16:128.36.232.43:50005 # newt -#node11=0:45:16:128.36.232.43:50005 # newt -#node11=0:46:16:128.36.232.43:50005 # newt -#node11=0:47:16:128.36.232.43:50005 # newt -#node12=0:48:16:128.36.232.28:50005 # termite -#node12=0:49:16:128.36.232.28:50005 # termite -#node12=0:50:16:128.36.232.28:50005 # termite -#node12=0:51:16:128.36.232.28:50005 # termite -#node13=0:52:16:128.36.232.46:50005 # python -#node13=0:53:16:128.36.232.46:50005 # python -#node13=0:54:16:128.36.232.46:50005 # python -#node13=0:55:16:128.36.232.46:50005 # python -#node14=0:56:16:128.36.232.5:50005 # cicada -#node14=0:57:16:128.36.232.5:50005 # cicada -#node14=0:58:16:128.36.232.5:50005 # cicada -#node14=0:59:16:128.36.232.5:50005 # cicada -#node15=0:60:16:128.36.232.35:50005 # aphid -#node15=0:61:16:128.36.232.35:50005 # aphid -#node15=0:62:16:128.36.232.35:50005 # aphid -#node15=0:63:16:128.36.232.35:50005 # aphid - - - - - - - - - - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_traditional_2pl_2pc/obj/.deps b/src_traditional_2pl_2pc/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_traditional_2pl_2pc/obj/common/configuration.d b/src_traditional_2pl_2pc/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_traditional_2pl_2pc/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_traditional_2pl_2pc/paxos/Makefile.inc b/src_traditional_2pl_2pc/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_traditional_2pl_2pc/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/paxos/paxos.cc b/src_traditional_2pl_2pc/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_traditional_2pl_2pc/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_traditional_2pl_2pc/paxos/paxos.h b/src_traditional_2pl_2pc/paxos/paxos.h deleted file mode 100755 index bbcd711d..00000000 --- a/src_traditional_2pl_2pc/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_traditional_2pl_2pc/paxos/zookeeper.conf b/src_traditional_2pl_2pc/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_traditional_2pl_2pc/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_traditional_2pl_2pc/proto/Makefile.inc b/src_traditional_2pl_2pc/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_traditional_2pl_2pc/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_traditional_2pl_2pc/proto/message.proto b/src_traditional_2pl_2pc/proto/message.proto deleted file mode 100755 index 91ddc8d9..00000000 --- a/src_traditional_2pl_2pc/proto/message.proto +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - PREPARED = 8; - PREPARED_REPLY = 9; - COMMIT = 10; - COMMIT_REPLY = 11; - TXN_RESTART = 12; - TXN_ABORT = 13; - WAIT_FOR_GRAPH = 14; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; - - optional int32 lock_or_release = 1003; - - - repeated int64 wait_txns = 2001; -} - diff --git a/src_traditional_2pl_2pc/proto/tpcc.proto b/src_traditional_2pl_2pc/proto/tpcc.proto deleted file mode 100755 index bc3143fb..00000000 --- a/src_traditional_2pl_2pc/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_traditional_2pl_2pc/proto/tpcc_args.proto b/src_traditional_2pl_2pc/proto/tpcc_args.proto deleted file mode 100755 index b0bf1122..00000000 --- a/src_traditional_2pl_2pc/proto/tpcc_args.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - repeated int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; - - - optional int32 order_number = 41; - - optional int32 lastest_order_number = 42; - optional int32 threshold = 51; -} diff --git a/src_traditional_2pl_2pc/proto/txn.proto b/src_traditional_2pl_2pc/proto/txn.proto deleted file mode 100755 index c8613b12..00000000 --- a/src_traditional_2pl_2pc/proto/txn.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; - - optional int32 txn_node = 50; - repeated int32 txn_other_node = 51; - -} - diff --git a/src_traditional_2pl_2pc/scheduler/Makefile.inc b/src_traditional_2pl_2pc/scheduler/Makefile.inc deleted file mode 100755 index 381173e0..00000000 --- a/src_traditional_2pl_2pc/scheduler/Makefile.inc +++ /dev/null @@ -1,23 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.cc b/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.cc deleted file mode 100755 index a745de62..00000000 --- a/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,601 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include -#include - -#include "scheduler/deterministic_lock_manager.h" -#include "sequencer/sequencer.h" - -#include - -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::vector; - -typedef DeterministicLockManager::Latch Latch; - -DeterministicLockManager::DeterministicLockManager(AtomicQueue* ready_txns, Configuration* config) - : configuration_(config), ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); - latches_ = new Latch[TABLE_SIZE]; - cond = new pthread_cond_t[WorkersNumber]; - mutex = new pthread_mutex_t[WorkersNumber]; - for(int i = 0; i < WorkersNumber; i++) { - pthread_cond_init(&cond[i], NULL); - pthread_mutex_init(&mutex[i], NULL); - } - pthread_mutex_init(&new_mutex_, NULL); - pthread_mutex_init(&distributed_deadlock_mutex_, NULL); -} - -Latch* DeterministicLockManager::LatchFor(const int& index) { - return latches_ + index; -} - -int DeterministicLockManager::Lock(TxnProto* txn, Connection* connection) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - int hash_index = Hash(txn->read_write_set(i)); - // Latch acquisitions - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_write_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - // Check whether there is a local deadlock - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - int hash_index = Hash(txn->read_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Record and return the number of locks that the txn is blocked on. - return not_acquired; -} - -int DeterministicLockManager::ContinueLock(TxnProto* txn, Connection* connection) { - int not_acquired = 0; - bool find_point = false; - pthread_mutex_lock(&new_mutex_); - Key point_key = txn_waits_[txn]; - pthread_mutex_unlock(&new_mutex_); - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - if (find_point == false) { - if (txn->read_write_set(i) == point_key) { - find_point = true; - continue; - } else { - continue; - } - } - - int hash_index = Hash(txn->read_write_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_write_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - if (find_point == false) { - if (txn->read_set(i) == point_key) { - find_point = true; - continue; - } else { - continue; - } - } - - int hash_index = Hash(txn->read_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - if (not_acquired == 0) { - pthread_mutex_lock(&new_mutex_); - txn_waits_.erase(txn); - pthread_mutex_unlock(&new_mutex_); - } - // Record and return the number of locks that the txn is blocked on. - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn, Connection* connection) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn, connection); - - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn, connection); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn, Connection* connection) { - int hash_index = Hash(key); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - - if (it1 == key_requests->end()) { - pthread_mutex_unlock(&latch->lock_); - return ; - } - - deque* requests = it1->locksrequest; - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - pthread_mutex_lock(&new_mutex_); - for (uint64 j = 0; j < new_owners.size(); j++) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - if (doing_deadlocks.count(IntToString(new_owners[j]->txn_id())) == 0) { - ready_txns_->Push(new_owners[j]); - } - - if (wait_for_graph.count(new_owners[j]->txn_id()) > 0) - wait_for_graph.erase(new_owners[j]->txn_id()); - - // Send message to machine 0, to delete blocked information from global wait for graph. - MessageProto message; - message.set_type(MessageProto::WAIT_FOR_GRAPH); - message.set_destination_channel("wait_for_graph"); - message.set_destination_node(0); - message.set_source_node(configuration_->this_node_id); - message.add_wait_txns(new_owners[j]->txn_id()); - message.add_wait_txns(txn->txn_id()); - message.set_lock_or_release(1); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - - } - pthread_mutex_unlock(&new_mutex_); - - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - } - pthread_mutex_unlock(&latch->lock_); -} - -// Local deadlock detection -bool DeterministicLockManager::CheckDeadlock(TxnProto* txn) { - bool deadlock = false; - int64 next_txn = wait_for_graph[txn->txn_id()]; - while (true) { - if (wait_for_graph.count(next_txn) == 0) { - deadlock = false; - break; - } else if (wait_for_graph[next_txn] == txn->txn_id()) { - deadlock = true; - break; - } - next_txn = wait_for_graph[next_txn]; - } - - return deadlock; -} - -// Distributed deadlock detection -bool DeterministicLockManager::CheckDistributedDeadlock(int64 txn1_id, int64 txn2_id) { - bool deadlock = false; - - vector aa = distributed_wait_for_graph[txn2_id]; -set tt; - for (int i = 0; i < (int)aa.size(); i ++) { - int64 bb = aa[i]; - if (bb == txn1_id) { - deadlock = true; - return deadlock; - } - - if (distributed_wait_for_graph.count(bb) > 0 && tt.count(bb) == 0) { - deadlock = CheckDistributedDeadlock(txn1_id, bb); - if (deadlock == true) { - return deadlock; - } - tt.insert(bb); - } - } - - return deadlock; -} - -void DeterministicLockManager::HandleDistributedDeadlock(MessageProto message, Connection* connection) { - - int lock_or_release = message.lock_or_release(); - - if (lock_or_release == 0) { - TxnProto* txn1 = new TxnProto(); - txn1->ParseFromString(message.data(0)); - int64 txn1_id = txn1->txn_id(); - int64 txn2_id = message.wait_txns(0); - - if (distributed_wait_for_graph.count(txn1_id) > 0) { - vector aa = distributed_wait_for_graph[txn1_id]; - for(int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].push_back(txn2_id); - return; - } - } - } - - distributed_wait_for_graph[txn1_id].push_back(txn2_id); - bool deadlock = CheckDistributedDeadlock(txn1_id, txn2_id); - if (deadlock == true) { - - vector aa = distributed_wait_for_graph[txn1_id]; - for (int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].erase(distributed_wait_for_graph[txn1_id].begin() + i); - } - } - if (distributed_wait_for_graph[txn1_id].size() == 0) { - distributed_wait_for_graph.erase(txn1_id); - } - -std::cout<<"!!!!!!: find a distributed deadlock, txn is "<txn_node()); - connection->Send(message1); - - for (int i = 0; i < txn1->txn_other_node_size(); i++) { - MessageProto message2; - message2.set_type(MessageProto::TXN_ABORT); - message2.set_destination_channel(IntToString(txn1_id)); - message2.set_destination_node(txn1->txn_other_node(i)); - connection->Send(message2); - } - - } - } else { - int64 txn1_id = message.wait_txns(0); - int64 txn2_id = message.wait_txns(1); - if (distributed_wait_for_graph.count(txn1_id) > 0) { - vector aa = distributed_wait_for_graph[txn1_id]; - for (int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].erase(distributed_wait_for_graph[txn1_id].begin() + i); - break; - } - } - if (distributed_wait_for_graph[txn1_id].size() == 0) { - distributed_wait_for_graph.erase(txn1_id); - } - } - } -} - -int DeterministicLockManager::try_deadlock_mutex() { - int ret = pthread_mutex_trylock(&distributed_deadlock_mutex_); - return ret; -} - -void DeterministicLockManager::release_deadlock_mutex() { - pthread_mutex_unlock(&distributed_deadlock_mutex_); -} - -TxnProto* DeterministicLockManager::GetBlockedTxn(string txn_id) { - pthread_mutex_lock(&new_mutex_); - TxnProto* txn = NULL; - for (unordered_map::iterator it = txn_waits_.begin(); - it != txn_waits_.end(); ++it) { - if (it->first->txn_id() == StringToInt(txn_id)) { - txn = it->first; - break; - } - } - - if (txn != NULL) { - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); - } - - pthread_mutex_unlock(&new_mutex_); - return txn; -} - - - diff --git a/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.h b/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.h deleted file mode 100755 index 3ba6dd69..00000000 --- a/src_traditional_2pl_2pc/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,115 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" -#include "common/connection.h" - -using std::tr1::unordered_map; -using std::deque; -using std::pair; -using std::vector; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(AtomicQueue* ready_txns, Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn, Connection* connection); - virtual int ContinueLock(TxnProto* txn, Connection* connection); - virtual void Release(TxnProto* txn, Connection* connection); - virtual void Release(const Key& key, TxnProto* txn, Connection* connection); - virtual bool CheckDeadlock(TxnProto* txn); - virtual bool CheckDistributedDeadlock(int64 txn1_id, int64 txn2_id); - virtual void HandleDistributedDeadlock(MessageProto message, Connection* connection); - virtual int try_deadlock_mutex(); - virtual void release_deadlock_mutex(); - virtual TxnProto* GetBlockedTxn(string txn_id); - - class Latch { - public: - pthread_mutex_t lock_; - - Latch() { - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const int& index); - -pthread_cond_t* cond; -pthread_mutex_t* mutex; -unordered_map txn_waits_; - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - //return atoi(key.c_str()); - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - - deque* lock_table_[TABLE_SIZE]; - - Latch* latches_; - - AtomicQueue* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - - pthread_mutex_t new_mutex_; - - pthread_mutex_t distributed_deadlock_mutex_; - - unordered_map wait_for_graph; - - unordered_map> distributed_wait_for_graph; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.cc b/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.cc deleted file mode 100755 index c4bd6435..00000000 --- a/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,505 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using std::bitset; - - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - ConnectionMultiplexer* multiplexer, - Storage* storage, - const Application* application) - : configuration_(conf), multiplexer_(multiplexer), storage_(storage){ - application_ = application; -pthread_mutex_init(&test_mutex_, NULL); -pthread_mutex_init(&batch_mutex_, NULL); - for (int i = 0; i < WorkersNumber; i++) { - message_queues[i] = new AtomicQueue(); - } - - batch_txns = new AtomicQueue(); - batch_id = 0; - - string channel_batch("scheduler_"); - batch_connections_ = multiplexer_->NewConnection(channel_batch); - -cpu_set_t cpuset; - - for (int i = 0; i < WorkersNumber; i++) { - string channel_worker("worker"); - channel_worker.append(IntToString(i)); - thread_connections_[i] = multiplexer_->NewConnection(channel_worker, &message_queues[i]); -CPU_ZERO(&cpuset); -pthread_attr_t attr; -pthread_attr_init(&attr); -if (i == 0 || i == 1) -CPU_SET(i, &cpuset); -else if (i == 2 || i == 3) -CPU_SET(i+2, &cpuset); -else -CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - pthread_create(&(threads_[i]), &attr, RunWorkerThread, reinterpret_cast( - new pair(configuration_->this_node_id * WorkersNumber + i, this))); - } - - ready_txns_ = new AtomicQueue(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - pthread_mutex_init(&deadlock_mutex_, NULL); - - if (configuration_->this_node_id == 0) { - string channel_worker("wait_for_graph"); - distributed_deadlock_queue = new AtomicQueue(); - distributed_deadlock_ = multiplexer_->NewConnection(channel_worker, &distributed_deadlock_queue); - } - - string channel_worker("force_abort_unactive_txn"); - abort_txn_queue = new AtomicQueue(); - abort_txn_ = multiplexer_->NewConnection(channel_worker, &abort_txn_queue); -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -int last_txn = -1;; -void GetBatch(DeterministicScheduler* scheduler) { - if(pthread_mutex_trylock(&scheduler->batch_mutex_) == 0) { - MessageProto* message = new MessageProto(); - if (scheduler->batch_connections_->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - batches[message->batch_number()] = message; - if (batches.count(scheduler->batch_id) > 0) { - message = batches[scheduler->batch_id]; - for (int i = 0; i < message->data_size(); i++) { - TxnProto* txn = new TxnProto(); - txn->ParseFromString(message->data(i)); - scheduler->batch_txns->Push(txn); - last_txn = txn->txn_id(); - } - batches.erase(scheduler->batch_id); - scheduler->batch_id ++; - delete message; - } - } else { - delete message; - } - - pthread_mutex_unlock(&scheduler->batch_mutex_); - } -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - - int partition_id = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - int thread = partition_id % WorkersNumber; - MessageProto message; - - int txns = 0; - double start_time = GetTime(); - double time = GetTime(); - - Storage* this_storage = scheduler->storage_; - int this_node_id = scheduler->configuration_->this_node_id; - - unordered_map active_txns; - - Spin(2); - - while(true) { - bool got_message = scheduler->abort_txn_queue->Pop(&message); - if (got_message == true) { - TxnProto* txn = scheduler->lock_manager_->GetBlockedTxn(message.destination_channel()); - if (txn == NULL) { - continue; - } - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - doing_deadlocks.erase(IntToString(txn->txn_id())); - if (txn->txn_node() == this_node_id) { - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->thread_connections_[thread]->Send(restart); - } - delete txn; - } - - got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - switch (message.type()) { - case MessageProto::TXN_ABORT: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - TxnProto* txn = manager->txn_; - assert(txn != NULL); - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->txn_node() == this_node_id) { - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } - doing_deadlocks.erase(IntToString(txn->txn_id())); - break; - } - - case MessageProto::READ_RESULT: - { - if (active_txns.count(message.destination_channel()) == 0) { - break; - } - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandleReadResult(message); - TxnProto* txn = manager->txn_; - if (manager->ReadyToExecute()) { - scheduler->application_->Execute(txn, manager); - // If this node is the coordinator, broadcast prepared messages - if (txn->txn_node() == this_node_id) { - manager->BroadcastPreparedMessages(); - } - if (manager->ReceivedPreparedMessages() == true) { - manager->SendPreparedReplyMessages(); - } - } - break; - } - - case MessageProto::PREPARED: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandlePreparedMessages(message); - if (manager->ReadyToExecute()) { - manager->SendPreparedReplyMessages(); - } - break; - } - - case MessageProto::PREPARED_REPLY: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandlePreparedReplyMessages(message); - if (manager->ReceivedAllPreparedReplyMessages()) { - manager->BroadcastCommitMessages(); - } - break; - } - - case MessageProto::COMMIT: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - TxnProto* txn = manager->txn_; - assert(txn != NULL); - manager->HandleCommitMessages(message); - manager->SendCommitReplyMessages(); - // Done the transaction - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if((txn->writers_size() == 0) && (rand() % txn->readers_size() == 0)) { - txns++; - } else if(rand() % txn->writers_size() == 0) { - txns++; - } - delete txn; - delete manager; - break; - } - - case MessageProto::COMMIT_REPLY: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - TxnProto* txn = manager->txn_; - manager->HandleCommitReplyMessages(message); - if (manager->ReceivedAllCommitReplyMessages()) { - // Done the transaction - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - - if((txn->writers_size() == 0) && (rand() % txn->readers_size() == 0)) { - txns++; - } else if(rand() % txn->writers_size() == 0) { - txns++; - } - delete txn; - delete manager; - } - break; - } - - default: - break; - } - - } - -if ((int)active_txns.size() < 100 && scheduler->lock_manager_->txn_waits_.size() < 40) { - TxnProto* txn; - bool got_message = scheduler->batch_txns->Pop(&txn); - if (got_message == true) { - if (doing_deadlocks.count(IntToString(txn->txn_id())) > 0) { - doing_deadlocks.erase(IntToString(txn->txn_id())); - continue; - } - - int ret = scheduler->lock_manager_->Lock(txn, scheduler->thread_connections_[thread]); - // Have acquired all locks, execute the txn now. - if(ret == 0) { - - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - this_storage, txn, this_node_id); - - if (manager->ReadyToExecute()) { - // Execute the txn. - scheduler->application_->Execute(txn, manager); - - // If don't happen deadlocks, just release all locks - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - delete manager; - delete txn; - txns++; - } else { - string channel = IntToString(txn->txn_id()); - scheduler->thread_connections_[thread]->LinkChannel(channel); - active_txns[channel] = manager; - } - - } else if (ret == -1) { - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->multipartition() == false) { - // Single-partition txn - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else if (txn->multipartition() == true) { - // Multi-partition txn - if (txn->txn_node() == this_node_id) { - for (int i = 0; i < txn->txn_other_node_size(); i++) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->thread_connections_[thread]->Send(restart); - } else { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_node()); - scheduler->thread_connections_[thread]->Send1(message); - - for (int i = 0; i < txn->txn_other_node_size(); i++) { - if (txn->txn_other_node(i) != this_node_id) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - } - } - - } - } - } -} - - - if (!scheduler->ready_txns_->Empty()) { - TxnProto* txn; - bool got_it = scheduler->ready_txns_->Pop(&txn); - if (got_it == true) { - if (doing_deadlocks.count(IntToString(txn->txn_id())) > 0) { - doing_deadlocks.erase(IntToString(txn->txn_id())); - continue; - } - int ret = scheduler->lock_manager_->ContinueLock(txn, scheduler->thread_connections_[thread]); - if (ret == 0) { - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - this_storage, txn, this_node_id); - if (manager->ReadyToExecute()) { - // Execute the txn. - scheduler->application_->Execute(txn, manager); - - // If don't happen deadlocks, just release all locks - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - - delete manager; - delete txn; - txns++; - } else { - string channel = IntToString(txn->txn_id()); - scheduler->thread_connections_[thread]->LinkChannel(channel); - active_txns[channel] = manager; - } - } else if (ret == -1) { - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->multipartition() == false) { - // Single-partition txn - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else if (txn->multipartition() == true) { - // Multi-partition txn - if (txn->txn_node() == this_node_id) { - for (int i = 0; i < txn->txn_other_node_size(); i++) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_node()); - scheduler->thread_connections_[thread]->Send1(message); - - for (int i = 0; i < txn->txn_other_node_size(); i++) { - if (txn->txn_other_node(i) != this_node_id) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - } - } - } - } - } - } - - if (this_node_id == 0 && scheduler->lock_manager_->try_deadlock_mutex() == 0) { - got_message = scheduler->distributed_deadlock_queue->Pop(&message); - if (got_message == true) { - scheduler->lock_manager_->HandleDistributedDeadlock(message, scheduler->thread_connections_[thread]); - scheduler->lock_manager_->release_deadlock_mutex(); - } else { - scheduler->lock_manager_->release_deadlock_mutex(); - } - } - - if (txns % 10 == 0) { - GetBatch(scheduler); - } - - if(GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout<<"Partition "<(txns)/total_time)<<" txns/sec. "<<(int)active_txns.size()<<" "<lock_manager_->txn_waits_.size()<<"\n"< start_time + 240) - exit(0); -} - - } - - diff --git a/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.h b/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.h deleted file mode 100755 index 5f43b3f7..00000000 --- a/src_traditional_2pl_2pc/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,89 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include -#include - -#include "scheduler/scheduler.h" -#include -#include "common/connection.h" -#include "common/configuration.h" -#include "scheduler/deterministic_lock_manager.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; -using std::tr1::unordered_map; -using std::atomic; - -class Configuration; -class Connection; -class ConnectionMultiplexer; -class DeterministicLockManager; -class Storage; -class TxnProto; -class Application; - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, ConnectionMultiplexer* multiplexer, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - // Connection for receiving txn batches from sequencer. - Connection* batch_connections_; - pthread_mutex_t test_mutex_; - pthread_mutex_t batch_mutex_; - AtomicQueue* batch_txns; - int batch_id; - - private: - // Application currently being run. - const Application* application_; - - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[WorkersNumber]; - Connection* thread_connections_[WorkersNumber]; - - ConnectionMultiplexer* multiplexer_; - - // Storage layer used in application execution. - Storage* storage_; - - DeterministicLockManager* lock_manager_; - - AtomicQueue* message_queues[WorkersNumber]; - - AtomicQueue* ready_txns_; - - atomic blocked; - - pthread_mutex_t deadlock_mutex_; - - Connection* distributed_deadlock_; - AtomicQueue* distributed_deadlock_queue; - - Connection* abort_txn_; - AtomicQueue* abort_txn_queue; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc/scheduler/lock_manager.h b/src_traditional_2pl_2pc/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_traditional_2pl_2pc/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc/scheduler/scheduler.h b/src_traditional_2pl_2pc/scheduler/scheduler.h deleted file mode 100755 index 45557a1b..00000000 --- a/src_traditional_2pl_2pc/scheduler/scheduler.h +++ /dev/null @@ -1,18 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc/scheduler/serial_scheduler.cc b/src_traditional_2pl_2pc/scheduler/serial_scheduler.cc deleted file mode 100755 index 1d2c9bfe..00000000 --- a/src_traditional_2pl_2pc/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_traditional_2pl_2pc/scheduler/serial_scheduler.h b/src_traditional_2pl_2pc/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_traditional_2pl_2pc/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc/sequencer/Makefile.inc b/src_traditional_2pl_2pc/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_traditional_2pl_2pc/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc/sequencer/sequencer.cc b/src_traditional_2pl_2pc/sequencer/sequencer.cc deleted file mode 100755 index fa2db69b..00000000 --- a/src_traditional_2pl_2pc/sequencer/sequencer.cc +++ /dev/null @@ -1,296 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST1 -double execute_begin[SAMPLES]; -double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(2, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - // Start Sequencer main loops running in background thread. - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(6, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { -std::cout << "~Sequencer() execute.\n" << std::flush; - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - if (connection_->GetMessage(&message)) { - txn = new TxnProto(); - assert(message.type() == MessageProto::TXN_RESTART); - txn->ParseFromString(message.data(0)); - txn->set_txn_id(batch_number * MAX_BATCH_SIZE + txn_id_offset); - // txn_id_offset++; - } else { - client_->GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); - if(txn->txn_id() == -1) { - delete txn; - continue; - } - // txn_id_offset++; - } - - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - txn_id_offset++; - delete txn; - } - } - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - // Set up batch messages for each system node. - map batches; - for (int i = 0; i < (int)configuration_->all_nodes.size(); i++) { - string channel("scheduler_"); - batches[i].set_destination_channel(channel); - batches[i].set_destination_node(i); - batches[i].set_type(MessageProto::TXN_BATCH); - } - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - - } while (!got_batch); -#endif - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - bytes txn_data; - txn.SerializeToString(&txn_data); - - - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) { - batches[*it].add_data(txn_data); - } - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - - // Report output. - if (GetTime() > time + 1) { - - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_traditional_2pl_2pc/sequencer/sequencer.h b/src_traditional_2pl_2pc/sequencer/sequencer.h deleted file mode 100755 index db7686d6..00000000 --- a/src_traditional_2pl_2pc/sequencer/sequencer.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -//#define VERBOSE_SEQUENCER -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 250 - -#define SAMPLES 10000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST1 - -using std::set; -using std::string; -using std::queue; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST1 -extern double execute_begin[SAMPLES]; -extern double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -extern unordered_map doing_deadlocks; - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_traditional_2pl_2pc/tests/collapsed_versioned_storage_test.cc b/src_traditional_2pl_2pc/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_traditional_2pl_2pc/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_traditional_2pl_2pc/tests/configuration_test.cc b/src_traditional_2pl_2pc/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_traditional_2pl_2pc/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_traditional_2pl_2pc/tests/connection_test.cc b/src_traditional_2pl_2pc/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_traditional_2pl_2pc/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_traditional_2pl_2pc/tests/deterministic_lock_manager_test.cc b/src_traditional_2pl_2pc/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_traditional_2pl_2pc/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_traditional_2pl_2pc/tests/deterministic_scheduler_test.cc b/src_traditional_2pl_2pc/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_traditional_2pl_2pc/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_traditional_2pl_2pc/tests/fetching_storage_test.cc b/src_traditional_2pl_2pc/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_traditional_2pl_2pc/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc/tests/microbenchmark_test.cc b/src_traditional_2pl_2pc/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_traditional_2pl_2pc/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_traditional_2pl_2pc/tests/paxos_test.cc b/src_traditional_2pl_2pc/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_traditional_2pl_2pc/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_traditional_2pl_2pc/tests/profile_test.cc b/src_traditional_2pl_2pc/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_traditional_2pl_2pc/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_traditional_2pl_2pc/tests/sequencer_test.cc b/src_traditional_2pl_2pc/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_traditional_2pl_2pc/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_traditional_2pl_2pc/tests/serial_scheduler_test.cc b/src_traditional_2pl_2pc/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_traditional_2pl_2pc/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_traditional_2pl_2pc/tests/simple_storage_test.cc b/src_traditional_2pl_2pc/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_traditional_2pl_2pc/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc/tests/storage_manager_test.cc b/src_traditional_2pl_2pc/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_traditional_2pl_2pc/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_traditional_2pl_2pc/tests/tpcc_test.cc b/src_traditional_2pl_2pc/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_traditional_2pl_2pc/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_traditional_2pl_2pc/tests/types_test.cc b/src_traditional_2pl_2pc/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_traditional_2pl_2pc/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_traditional_2pl_2pc_3_partitions/.cproject b/src_traditional_2pl_2pc_3_partitions/.cproject deleted file mode 100755 index 61ad6f76..00000000 --- a/src_traditional_2pl_2pc_3_partitions/.cproject +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src_traditional_2pl_2pc_3_partitions/.project b/src_traditional_2pl_2pc_3_partitions/.project deleted file mode 100755 index 590162ae..00000000 --- a/src_traditional_2pl_2pc_3_partitions/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - src - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - all - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/src_traditional_2pl_2pc_3_partitions/Makefile b/src_traditional_2pl_2pc_3_partitions/Makefile deleted file mode 100755 index fc13f190..00000000 --- a/src_traditional_2pl_2pc_3_partitions/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror -Wno-strict-aliasing -std=c++0x -fno-omit-frame-pointer - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_traditional_2pl_2pc_3_partitions/Makefile.template b/src_traditional_2pl_2pc_3_partitions/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_traditional_2pl_2pc_3_partitions/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_traditional_2pl_2pc_3_partitions/applications/.DS_Store b/src_traditional_2pl_2pc_3_partitions/applications/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/applications/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/applications/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/applications/application.h b/src_traditional_2pl_2pc_3_partitions/applications/application.h deleted file mode 100755 index 711284fa..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" -#include "backend/simple_storage.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.cc b/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.cc deleted file mode 100755 index 5d6841ec..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.cc +++ /dev/null @@ -1,173 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -int cmp(const void *a, const void *b) { - return *(int *)a - *(int *)b; -} - -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - txn->set_multipartition(false); - txn->set_txn_node(part); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - return txn; -} - - -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2, int part3) { - assert(part1 != part2 && part3 != part1 && part3 != part2); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - txn->set_multipartition(true); - txn->set_txn_node(part1); - txn->add_txn_other_node(part2); - txn->add_txn_other_node(part3); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - int hotkey3 = part3 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - txn->add_read_write_set(IntToString(hotkey3)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - 3, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part3); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - - *val = IntToString(StringToInt(*val) + 1); - - // The following is for "long" transaction - int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - } - - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, Configuration* conf) const { - for (int i = 0; i < nparts * kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { - storage->PutObject(IntToString(i), new Value(IntToString(i))); - } - } -} - diff --git a/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.h b/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.h deleted file mode 100755 index 430571d2..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/microbenchmark.h +++ /dev/null @@ -1,55 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" -#include "backend/simple_storage.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part1); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2, int part3); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/applications/tpcc.cc b/src_traditional_2pl_2pc_3_partitions/applications/tpcc.cc deleted file mode 100644 index fa29cd36..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/tpcc.cc +++ /dev/null @@ -1,1117 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; -// if (rand() / (static_cast(RAND_MAX + 1.0)) <= 0.00) -// invalid = true; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s",warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s",district_string.c_str()); - - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s",customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d",warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d",warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), - "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", - order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", - warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])){ - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - txn->add_read_write_set(customer_key); - } - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if(storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", - warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // double customer_balance = customer->balance(); - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - assert(order->ParseFromString(*order_value)); - // int carrier_id = order->carrier_id(); - // double entry_date = order->entry_date(); - - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - // int quantity = order_line->quantity(); - // double amount = order_line->amount(); - // double delivery_date = order_line->delivery_date(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_traditional_2pl_2pc_3_partitions/applications/tpcc.h b/src_traditional_2pl_2pc_3_partitions/applications/tpcc.h deleted file mode 100644 index b01657e2..00000000 --- a/src_traditional_2pl_2pc_3_partitions/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 12 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/backend/.DS_Store b/src_traditional_2pl_2pc_3_partitions/backend/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/backend/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/backend/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/backend/Makefile.inc deleted file mode 100755 index edcf67bd..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/Makefile.inc +++ /dev/null @@ -1,28 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/checkpointable_storage.cc \ - backend/collapsed_versioned_storage.cc \ - backend/fetching_storage.cc \ - backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.cc b/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.h b/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.h deleted file mode 100755 index 249a40d6..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.cc b/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.cc deleted file mode 100755 index f5514372..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.h b/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.h deleted file mode 100755 index 273dc04c..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.cc b/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.cc deleted file mode 100755 index 46237c8f..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { -pthread_mutex_lock(&mutex_); - objects_[key] = value; -pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.h b/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.h deleted file mode 100755 index 9fb2ee24..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/backend/storage.h b/src_traditional_2pl_2pc_3_partitions/backend/storage.h deleted file mode 100755 index a793360a..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/storage.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.cc b/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.cc deleted file mode 100755 index 7de4ff40..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == configuration_->this_node_id) - reader = true; - } - - if (reader) { - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == - configuration_->this_node_id) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != configuration_->this_node_id) { - message.set_destination_node(txn->writers(i)); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == configuration_->this_node_id) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == configuration_->this_node_id) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.h b/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.h deleted file mode 100755 index 6d02dfc4..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/storage_manager.h +++ /dev/null @@ -1,87 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/backend/versioned_storage.h b/src_traditional_2pl_2pc_3_partitions/backend/versioned_storage.h deleted file mode 100755 index 24c6223b..00000000 --- a/src_traditional_2pl_2pc_3_partitions/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/common/.DS_Store b/src_traditional_2pl_2pc_3_partitions/common/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/common/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/common/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/common/configuration.cc b/src_traditional_2pl_2pc_3_partitions/common/configuration.cc deleted file mode 100755 index a55c0b81..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/configuration.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "common/utils.h" - -using std::string; - -Configuration::Configuration(int node_id, const string& filename) - : this_node_id(node_id) { - if (ReadFromFile(filename)) // Reading from file failed. - exit(0); -} - -// TODO(alex): Implement better (application-specific?) partitioning. -int Configuration::LookupPartition(const Key& key) const { - if (key.find("w") == 0) // TPCC - return OffsetStringToInt(key, 1) % static_cast(all_nodes.size()); - else - return StringToInt(key) % static_cast(all_nodes.size()); -} - -bool Configuration::WriteToFile(const string& filename) const { - FILE* fp = fopen(filename.c_str(), "w"); - if (fp == NULL) - return false; - for (map::const_iterator it = all_nodes.begin(); - it != all_nodes.end(); ++it) { - Node* node = it->second; - fprintf(fp, "node%d=%d:%d:%d:%s:%d\n", - it->first, - node->replica_id, - node->partition_id, - node->cores, - node->host.c_str(), - node->port); - } - fclose(fp); - return true; -} - -int Configuration::ReadFromFile(const string& filename) { - char buf[1024]; - FILE* fp = fopen(filename.c_str(), "r"); - if (fp == NULL) { - printf("Cannot open config file %s\n", filename.c_str()); - return -1; - } - char* tok; - // Loop through all lines in the file. - while (fgets(buf, sizeof(buf), fp)) { - // Seek to the first non-whitespace character in the line. - char* p = buf; - while (isspace(*p)) - ++p; - // Skip comments & blank lines. - if (*p == '#' || *p == '\0') - continue; - // Process the rest of the line, which has the format "=". - char* key = strtok_r(p, "=\n", &tok); - char* value = strtok_r(NULL, "=\n", &tok); - ProcessConfigLine(key, value); - } - fclose(fp); - return 0; -} - -void Configuration::ProcessConfigLine(char key[], char value[]) { - if (strncmp(key, "node", 4) != 0) { -#if VERBOSE - printf("Unknown key in config file: %s\n", key); -#endif - } else { - Node* node = new Node(); - // Parse node id. - node->node_id = atoi(key + 4); - - // Parse additional node addributes. - char* tok; - node->replica_id = atoi(strtok_r(value, ":", &tok)); - node->partition_id = atoi(strtok_r(NULL, ":", &tok)); - node->cores = atoi(strtok_r(NULL, ":", &tok)); - const char* host = strtok_r(NULL, ":", &tok); - node->port = atoi(strtok_r(NULL, ":", &tok)); - - // Translate hostnames to IP addresses. - string ip; - { - struct hostent* ent = gethostbyname(host); - if (ent == NULL) { - ip = host; - } else { - uint32_t n; - char buf[32]; - memmove(&n, ent->h_addr_list[0], ent->h_length); - n = ntohl(n); - snprintf(buf, sizeof(buf), "%u.%u.%u.%u", - n >> 24, (n >> 16) & 0xff, - (n >> 8) & 0xff, n & 0xff); - ip = buf; - } - } - node->host = ip; - - all_nodes[node->node_id] = node; - } -} - diff --git a/src_traditional_2pl_2pc_3_partitions/common/configuration.h b/src_traditional_2pl_2pc_3_partitions/common/configuration.h deleted file mode 100755 index d7ce7fb5..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/configuration.h +++ /dev/null @@ -1,95 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/types.h" - - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - -struct Node { - // Globally unique node identifier. - int node_id; - int replica_id; - int partition_id; - - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/common/configuration_test.conf b/src_traditional_2pl_2pc_3_partitions/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_traditional_2pl_2pc_3_partitions/common/configuration_test_one_node.conf b/src_traditional_2pl_2pc_3_partitions/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_traditional_2pl_2pc_3_partitions/common/connection.cc b/src_traditional_2pl_2pc_3_partitions/common/connection.cc deleted file mode 100755 index d6a1e7d5..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/connection.cc +++ /dev/null @@ -1,379 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - -send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - - if ((new_connection_channel_->substr(0, 9) == "scheduler") && (new_connection_channel_->substr(9,1) != "_")) { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - - if (message.type() == MessageProto::READ_RESULT) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/common/connection.h b/src_traditional_2pl_2pc_3_partitions/common/connection.h deleted file mode 100755 index 99fa7c0e..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/connection.h +++ /dev/null @@ -1,197 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/common/test_template.cc b/src_traditional_2pl_2pc_3_partitions/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/common/testing.h b/src_traditional_2pl_2pc_3_partitions/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/common/types.h b/src_traditional_2pl_2pc_3_partitions/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/common/utils.h b/src_traditional_2pl_2pc_3_partitions/common/utils.h deleted file mode 100755 index d5dd187b..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_traditional_2pl_2pc_3_partitions/common/zmq.hpp b/src_traditional_2pl_2pc_3_partitions/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_traditional_2pl_2pc_3_partitions/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/.DS_Store b/src_traditional_2pl_2pc_3_partitions/deployment/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/deployment/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/addhosts.pl b/src_traditional_2pl_2pc_3_partitions/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/cluster.cc b/src_traditional_2pl_2pc_3_partitions/deployment/cluster.cc deleted file mode 100755 index c04f82b9..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/cluster.cc +++ /dev/null @@ -1,378 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/main.cc b/src_traditional_2pl_2pc_3_partitions/deployment/main.cc deleted file mode 100755 index c9fe4c14..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/main.cc +++ /dev/null @@ -1,191 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "backend/simple_storage.h" -#include "backend/fetching_storage.h" -#include "backend/collapsed_versioned_storage.h" -#include "scheduler/serial_scheduler.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -map latest_order_id_for_customer; -map latest_order_id_for_district; -map smallest_order_id_for_district; -map customer_for_order; -unordered_map next_order_id_for_district; -map item_for_order_line; -map order_line_number; - -vector* involed_customers; - -pthread_mutex_t mutex_; -pthread_mutex_t mutex_for_item; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other1; - int other2; - int other3; - do { - other1 = rand() % config_->all_nodes.size(); - } while (other1 == config_->this_node_id); - - do { - other2 = rand() % config_->all_nodes.size(); - } while (other2 == config_->this_node_id || other2 == other1); - - do { - other3 = rand() % config_->all_nodes.size(); - } while (other3 == config_->this_node_id || other3 == other1 || other3 == other2); - - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other1, other2, other3); - } else { - // Single-partition txn. - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - int random_txn_type = rand() % 100; - // New order txn - if (random_txn_type < 45) { - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - } else if(random_txn_type < 88) { - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } else if(random_txn_type < 92) { - *txn = tpcc.NewTxn(txn_id, TPCC::ORDER_STATUS, args_string, config_); - args.set_multipartition(false); - } else if(random_txn_type < 96){ - *txn = tpcc.NewTxn(txn_id, TPCC::DELIVERY, args_string, config_); - args.set_multipartition(false); - } else { - *txn = tpcc.NewTxn(txn_id, TPCC::STOCK_LEVEL, args_string, config_); - args.set_multipartition(false); - } - - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - bool useFetching = false; - if (argc > 4 && argv[4][0] == 'f') - useFetching = true; - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif -pthread_mutex_init(&mutex_, NULL); -pthread_mutex_init(&mutex_for_item, NULL); -involed_customers = new vector; - - Storage* storage; - if (!useFetching) { - storage = new SimpleStorage(); - } else { - storage = FetchingStorage::BuildStorage(); - } -storage->Initmutex(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client, - storage); - - // Run scheduler in main thread. - if (argv[2][0] == 'm') { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - DeterministicScheduler scheduler(&config, - multiplexer.NewConnection("scheduler_"), - storage, - new TPCC()); - } - - Spin(180); - return 0; -} - diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/makeconf.pl b/src_traditional_2pl_2pc_3_partitions/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/portfile b/src_traditional_2pl_2pc_3_partitions/deployment/portfile deleted file mode 100755 index 7436f00b..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -54564 diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/run.pbs b/src_traditional_2pl_2pc_3_partitions/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_traditional_2pl_2pc_3_partitions/deployment/test.conf b/src_traditional_2pl_2pc_3_partitions/deployment/test.conf deleted file mode 100755 index 36bc2882..00000000 --- a/src_traditional_2pl_2pc_3_partitions/deployment/test.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Node=:::: - -#node0=0:0:16:128.36.232.50:50005 # tick - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:1:12:128.36.232.12:51261 #rhino -node2=0:2:12:128.36.232.15:51262 # gator -node3=0:3:12:128.36.232.20:51261 #macaw -node4=0:4:12:128.36.232.19:51262 # peacock -node5=0:5:12:128.36.232.34:51261 # perch -node6=0:6:12:128.36.232.14:51262 # jaguar -node7=0:7:12:128.36.232.16:51261 # hippo -node8=0:8:12:128.36.232.39:51261 #viper -node9=0:9:12:128.36.232.7:51261 #hornet -node10=0:10:12:128.36.232.8:51261 #cricket -node11=0:11:12:128.36.232.28:51261 #termite -node12=0:12:12:128.36.232.5:51261 #cicada -node13=0:13:12:128.36.232.6:51261 #bumblebee -node14=0:14:12:128.36.232.44:51261 #frog -node15=0:15:12:128.36.232.50:51261 #tick -node16=0:16:12:128.36.232.42:51261 #turtle -node17=0:17:12:128.36.232.35:51261 #aphid -node18=0:18:12:128.36.232.41:51261 #chameleon -node19=0:19:12:128.36.232.38:51261 #rattlesnake -node20=0:20:12:128.36.232.45:51261 #scorpion -node21=0:21:12:128.36.232.29:51261 #cardinal -node22=0:22:12:128.36.232.24:51261 #swan -node23=0:23:12:128.36.232.27:51261 #monkey - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_traditional_2pl_2pc_3_partitions/obj/.deps b/src_traditional_2pl_2pc_3_partitions/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.d b/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.o b/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.o deleted file mode 100755 index 510cfb78..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/obj/common/configuration.o and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/paxos/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_traditional_2pl_2pc_3_partitions/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/paxos/paxos.cc b/src_traditional_2pl_2pc_3_partitions/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_traditional_2pl_2pc_3_partitions/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_traditional_2pl_2pc_3_partitions/paxos/paxos.h b/src_traditional_2pl_2pc_3_partitions/paxos/paxos.h deleted file mode 100755 index 8488953e..00000000 --- a/src_traditional_2pl_2pc_3_partitions/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/paxos/zookeeper.conf b/src_traditional_2pl_2pc_3_partitions/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_traditional_2pl_2pc_3_partitions/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_traditional_2pl_2pc_3_partitions/proto/.DS_Store b/src_traditional_2pl_2pc_3_partitions/proto/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/proto/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/proto/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_traditional_2pl_2pc_3_partitions/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_traditional_2pl_2pc_3_partitions/proto/message.proto b/src_traditional_2pl_2pc_3_partitions/proto/message.proto deleted file mode 100755 index 1ee70946..00000000 --- a/src_traditional_2pl_2pc_3_partitions/proto/message.proto +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - PREPARED = 8; - PREPARED_REPLY = 9; - COMMIT = 10; - COMMIT_REPLY = 11; - TXN_RESTART = 12; - TXN_ABORT = 13; - WAIT_FOR_GRAPH = 14; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; - - optional int32 lock_or_release = 1003; - - - repeated int64 wait_txns = 2001; -} - diff --git a/src_traditional_2pl_2pc_3_partitions/proto/tpcc.proto b/src_traditional_2pl_2pc_3_partitions/proto/tpcc.proto deleted file mode 100755 index 50088661..00000000 --- a/src_traditional_2pl_2pc_3_partitions/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_traditional_2pl_2pc_3_partitions/proto/tpcc_args.proto b/src_traditional_2pl_2pc_3_partitions/proto/tpcc_args.proto deleted file mode 100755 index 142055e5..00000000 --- a/src_traditional_2pl_2pc_3_partitions/proto/tpcc_args.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - optional int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; -} diff --git a/src_traditional_2pl_2pc_3_partitions/proto/txn.proto b/src_traditional_2pl_2pc_3_partitions/proto/txn.proto deleted file mode 100755 index 3003fb13..00000000 --- a/src_traditional_2pl_2pc_3_partitions/proto/txn.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; - - optional int32 txn_node = 50; - repeated int32 txn_other_node = 51; - -} - diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/.DS_Store b/src_traditional_2pl_2pc_3_partitions/scheduler/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/scheduler/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/scheduler/Makefile.inc deleted file mode 100755 index f5678dcc..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc \ - scheduler/serial_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.cc b/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.cc deleted file mode 100644 index 15d002e3..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include "scheduler/deterministic_lock_manager.h" - -#include - -#include "proto/txn.pb.h" - -using std::vector; - -DeterministicLockManager::DeterministicLockManager( - deque* ready_txns, - Configuration* config) - : configuration_(config), - ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); -} - -int DeterministicLockManager::Lock(TxnProto* txn) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_write_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) - not_acquired++; - } - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - deque* key_requests = lock_table_[Hash(txn->read_set(i))]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - break; - } - } - } - } - } - - // Record and return the number of locks that the txn is blocked on. - if (not_acquired > 0) - txn_waits_[txn] = not_acquired; - else - ready_txns_->push_back(txn); - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn); - // Currently commented out because nothing in any write set can conflict - // in TPCC or Microbenchmark. -// for (int i = 0; i < txn->write_set_size(); i++) -// if (IsLocal(txn->write_set(i))) -// Release(txn->write_set(i), txn); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn) { - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[Hash(key)]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - deque* requests = it1->locksrequest; - - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - for (uint64 j = 0; j < new_owners.size(); j++) { - txn_waits_[new_owners[j]]--; - if (txn_waits_[new_owners[j]] == 0) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - ready_txns_->push_back(new_owners[j]); - txn_waits_.erase(new_owners[j]); - } - } - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - - } -} - diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.h b/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.h deleted file mode 100755 index aadd3688..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" - -using std::tr1::unordered_map; -using std::deque; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(deque* ready_txns, - Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn); - virtual void Release(const Key& key, TxnProto* txn); - virtual void Release(TxnProto* txn); - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - deque* lock_table_[TABLE_SIZE]; - - // Queue of pointers to transactions that have acquired all locks that - // they have requested. 'ready_txns_[key].front()' is the owner of the lock - // for a specified key. - // - // Owned by the DeterministicScheduler. - deque* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - unordered_map txn_waits_; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.cc b/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.cc deleted file mode 100755 index fb49b927..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,298 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/zmq.hpp" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" -#include "applications/tpcc.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using zmq::socket_t; -using std::map; - -static void DeleteTxnPtr(void* data, void* hint) { free(data); } - -void DeterministicScheduler::SendTxnPtr(socket_t* socket, TxnProto* txn) { - TxnProto** txn_ptr = reinterpret_cast(malloc(sizeof(txn))); - *txn_ptr = txn; - zmq::message_t msg(txn_ptr, sizeof(*txn_ptr), DeleteTxnPtr, NULL); - socket->send(msg); -} - -TxnProto* DeterministicScheduler::GetTxnPtr(socket_t* socket, - zmq::message_t* msg) { - if (!socket->recv(msg, ZMQ_NOBLOCK)) - return NULL; - TxnProto* txn = *reinterpret_cast(msg->data()); - return txn; -} - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - Connection* batch_connection, - Storage* storage, - const Application* application) - : configuration_(conf), batch_connection_(batch_connection), - storage_(storage), application_(application) { - ready_txns_ = new std::deque(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - txns_queue = new AtomicQueue(); - done_queue = new AtomicQueue(); - - for (int i = 0; i < NUM_THREADS; i++) { - message_queues[i] = new AtomicQueue(); - } - -Spin(2); - - // start lock manager thread - cpu_set_t cpuset; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - //pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - - CPU_ZERO(&cpuset); - CPU_SET(7, &cpuset); - pthread_attr_setaffinity_np(&attr1, sizeof(cpu_set_t), &cpuset); - pthread_create(&lock_manager_thread_, &attr1, LockManagerThread, - reinterpret_cast(this)); - - - // Start all worker threads. - for (int i = 0; i < NUM_THREADS; i++) { - string channel("scheduler"); - channel.append(IntToString(i)); - thread_connections_[i] = batch_connection_->multiplexer()->NewConnection(channel, &message_queues[i]); - - pthread_attr_t attr; - pthread_attr_init(&attr); - CPU_ZERO(&cpuset); - if (i == 0 || i == 1) - CPU_SET(i, &cpuset); - else - CPU_SET(i+2, &cpuset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - pthread_create(&(threads_[i]), &attr, RunWorkerThread, - reinterpret_cast( - new pair(i, this))); - } - -} - -void UnfetchAll(Storage* storage, TxnProto* txn) { - for (int i = 0; i < txn->read_set_size(); i++) - if (StringToInt(txn->read_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_set(i)); - for (int i = 0; i < txn->read_write_set_size(); i++) - if (StringToInt(txn->read_write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->read_write_set(i)); - for (int i = 0; i < txn->write_set_size(); i++) - if (StringToInt(txn->write_set(i)) > COLD_CUTOFF) - storage->Unfetch(txn->write_set(i)); -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - int thread = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - unordered_map active_txns; - - // Begin main loop. - MessageProto message; - while (true) { - bool got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - // Remote read result. - assert(message.type() == MessageProto::READ_RESULT); - StorageManager* manager = active_txns[message.destination_channel()]; - manager->HandleReadResult(message); - if (manager->ReadyToExecute()) { - // Execute and clean up. - TxnProto* txn = manager->txn_; - scheduler->application_->Execute(txn, manager); - delete manager; - - scheduler->thread_connections_[thread]-> - UnlinkChannel(IntToString(txn->txn_id())); - active_txns.erase(message.destination_channel()); - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } - } else { - // No remote read result found, start on next txn if one is waiting. - TxnProto* txn; - bool got_it = scheduler->txns_queue->Pop(&txn); - if (got_it == true) { - // Create manager. - StorageManager* manager = - new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - scheduler->storage_, txn); - - // Writes occur at this node. - if (manager->ReadyToExecute()) { - // No remote reads. Execute and clean up. - scheduler->application_->Execute(txn, manager); - delete manager; - - // Respond to scheduler; - //scheduler->SendTxnPtr(scheduler->responses_out_[thread], txn); - scheduler->done_queue->Push(txn); - } else { - scheduler->thread_connections_[thread]-> - LinkChannel(IntToString(txn->txn_id())); - // There are outstanding remote reads. - active_txns[IntToString(txn->txn_id())] = manager; - } - } - } - } - return NULL; -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -MessageProto* GetBatch(int batch_id, Connection* connection) { - if (batches.count(batch_id) > 0) { - // Requested batch has already been received. - MessageProto* batch = batches[batch_id]; - batches.erase(batch_id); - return batch; - } else { - MessageProto* message = new MessageProto(); - while (connection->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - if (message->batch_number() == batch_id) { - return message; - } else { - batches[message->batch_number()] = message; - message = new MessageProto(); - } - } - delete message; - return NULL; - } -} - -void* DeterministicScheduler::LockManagerThread(void* arg) { - DeterministicScheduler* scheduler = reinterpret_cast(arg); - - // Run main loop. - MessageProto message; - MessageProto* batch_message = NULL; - int txns = 0; - double time = GetTime(); - int executing_txns = 0; - int pending_txns = 0; - int batch_offset = 0; - int batch_number = 0; -//int test = 0; - while (true) { - TxnProto* done_txn; - bool got_it = scheduler->done_queue->Pop(&done_txn); - if (got_it == true) { - // We have received a finished transaction back, release the lock - scheduler->lock_manager_->Release(done_txn); - executing_txns--; - - if(done_txn->writers_size() == 0 || rand() % done_txn->writers_size() == 0) - txns++; - - delete done_txn; - - } else { - // Have we run out of txns in our batch? Let's get some new ones. - if (batch_message == NULL) { - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Done with current batch, get next. - } else if (batch_offset >= batch_message->data_size()) { - batch_offset = 0; - batch_number++; - delete batch_message; - batch_message = GetBatch(batch_number, scheduler->batch_connection_); - - // Current batch has remaining txns, grab up to 10. - } else if (executing_txns + pending_txns < 2000) { - - for (int i = 0; i < 100; i++) { - if (batch_offset >= batch_message->data_size()) { - // Oops we ran out of txns in this batch. Stop adding txns for now. - break; - } - TxnProto* txn = new TxnProto(); - txn->ParseFromString(batch_message->data(batch_offset)); - batch_offset++; - - scheduler->lock_manager_->Lock(txn); - pending_txns++; - } - - } - } - - // Start executing any and all ready transactions to get them off our plate - while (!scheduler->ready_txns_->empty()) { - TxnProto* txn = scheduler->ready_txns_->front(); - scheduler->ready_txns_->pop_front(); - pending_txns--; - executing_txns++; - scheduler->txns_queue->Push(txn); - //scheduler->SendTxnPtr(scheduler->requests_out_, txn); - - } - - // Report throughput. - if (GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout << "Completed " << (static_cast(txns) / total_time) - << " txns/sec, " - //<< test<< " for drop speed , " - << executing_txns << " executing, " - << pending_txns << " pending\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - //test ++; - } - } - return NULL; -} diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/lock_manager.h b/src_traditional_2pl_2pc_3_partitions/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/scheduler.h b/src_traditional_2pl_2pc_3_partitions/scheduler/scheduler.h deleted file mode 100755 index d93465fa..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/scheduler.h +++ /dev/null @@ -1,19 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} - -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.cc b/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.cc deleted file mode 100755 index 3e532069..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun.ren@yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.h b/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_traditional_2pl_2pc_3_partitions/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/sequencer/.DS_Store b/src_traditional_2pl_2pc_3_partitions/sequencer/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_3_partitions/sequencer/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_3_partitions/sequencer/Makefile.inc b/src_traditional_2pl_2pc_3_partitions/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_traditional_2pl_2pc_3_partitions/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.cc b/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.cc deleted file mode 100755 index dbd2b2e7..00000000 --- a/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.cc +++ /dev/null @@ -1,305 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST1 -double execute_begin[SAMPLES]; -double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(2, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - // Start Sequencer main loops running in background thread. - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(6, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { -std::cout << "~Sequencer() execute.\n" << std::flush; - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - if (connection_->GetMessage(&message)) { - txn = new TxnProto(); - assert(message.type() == MessageProto::TXN_RESTART); - txn->ParseFromString(message.data(0)); - txn->set_txn_id(batch_number * MAX_BATCH_SIZE + txn_id_offset); - txn_id_offset++; -std::cout<<"~~~~ Sequencer received a MessageProto::TXN_RESTART message, txn id is "<txn_id()<<"\n"<GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); - txn_id_offset++; - } - - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } - - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif -//std::cout << "Starting sequencer, enter RunReader.\n" << std::flush; - // Set up batch messages for each system node. - map batches; - for (int i = 0; i < (int)configuration_->all_nodes.size(); i++) { -//printf("TODO-----\n"); - string channel("scheduler_"); - batches[i].set_destination_channel(channel); - batches[i].set_destination_node(i); - batches[i].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - while (!deconstructor_invoked_) { - - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - - pthread_mutex_unlock(&mutex_); - - if (!got_batch) - Spin(0.001); - - } while (!got_batch); -#endif - - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - bytes txn_data; - txn.SerializeToString(&txn_data); - - - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) { - batches[*it].add_data(txn_data); - } - - txn_count++; - } - - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - - // Report output. - if (GetTime() > time + 1) { - - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.h b/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.h deleted file mode 100755 index 3b75b3e9..00000000 --- a/src_traditional_2pl_2pc_3_partitions/sequencer/sequencer.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -//#define VERBOSE_SEQUENCER -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 25 - -#define SAMPLES 10000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST1 - -using std::set; -using std::string; -using std::queue; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST1 -extern double execute_begin[SAMPLES]; -extern double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -extern unordered_map doing_deadlocks; - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_traditional_2pl_2pc_3_partitions/tests/collapsed_versioned_storage_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/configuration_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/connection_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/deterministic_lock_manager_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/deterministic_scheduler_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_traditional_2pl_2pc_3_partitions/tests/fetching_storage_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/microbenchmark_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/paxos_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_traditional_2pl_2pc_3_partitions/tests/profile_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_traditional_2pl_2pc_3_partitions/tests/sequencer_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/serial_scheduler_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_traditional_2pl_2pc_3_partitions/tests/simple_storage_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/storage_manager_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/tpcc_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_traditional_2pl_2pc_3_partitions/tests/types_test.cc b/src_traditional_2pl_2pc_3_partitions/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_traditional_2pl_2pc_3_partitions/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -} diff --git a/src_traditional_2pl_2pc_4_partitions/.cproject b/src_traditional_2pl_2pc_4_partitions/.cproject deleted file mode 100755 index 61ad6f76..00000000 --- a/src_traditional_2pl_2pc_4_partitions/.cproject +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src_traditional_2pl_2pc_4_partitions/.project b/src_traditional_2pl_2pc_4_partitions/.project deleted file mode 100755 index 590162ae..00000000 --- a/src_traditional_2pl_2pc_4_partitions/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - src - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - all - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/src_traditional_2pl_2pc_4_partitions/Makefile b/src_traditional_2pl_2pc_4_partitions/Makefile deleted file mode 100755 index fc13f190..00000000 --- a/src_traditional_2pl_2pc_4_partitions/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# This makefile system follows the structuring conventions -# recommended by Peter Miller in his excellent paper: -# -# Recursive Make Considered Harmful -# http://aegis.sourceforge.net/auug97.pdf -# - -# We create a listing of the root directories for access into -OBJDIR := ../obj -BINDIR := ../bin -EXTDIR := ../ext -SRCDIR := ../src -DBDIR := ../db -CHKPNT := ../db/checkpoints -STORE := ../db/storage - -# Next, we define the external libraries we use and their directories -ZEROMQ := $(EXTDIR)/zeromq -PROTOB := $(EXTDIR)/protobuf -GTEST := $(EXTDIR)/googletest -ZOOKPR := $(EXTDIR)/zookeeper - -# Executable for protocol buffers -PROTOCEXE := $(PROTOB)/src/.libs/protoc - -# '$(V)' controls whether the lab makefiles print verbose commands (the -# actual shell commands run by Make), as well as the "overview" commands -# (such as '+ cc lib/readline.c'). -# -# For overview commands only, the line should read 'V = @'. -# For overview and verbose commands, the line should read 'V ='. -V = @ - -# Set the compiler and compile-time loaded libraries -CXX := g++ -LDLIBRARYPATH := LD_LIBRARY_PATH=$(ZEROMQ)/src/.libs:$(PROTOB)/src/.libs:$(GTEST)/lib/.libs:$(ZOOKPR)/.libs - -# For profiling (with gprof), this line should read 'PG = -pg' -# For fast execution, this line should read 'PG ='. -PG = - -# Set the flags for C++ to compile with (namely where to look for external -# libraries) and the linker libraries (again to look in the ext/ library) -CXXFLAGS := -g -MD $(PG) -I$(SRCDIR) -I$(OBJDIR) \ - -I$(ZEROMQ)/include \ - -I$(PROTOB)/src \ - -I$(GTEST)/include \ - -I$(ZOOKPR)/include -I$(ZOOKPR)/generated -CXXFLAGS += -Wall -Werror -Wno-strict-aliasing -std=c++0x -fno-omit-frame-pointer - -LDFLAGS := -lrt -lpthread $(PG) \ - -lprotobuf -L$(PROTOB)/src/.libs \ - -lgtest -L$(GTEST)/lib/.libs \ - -lzookeeper_mt -L$(ZOOKPR)/.libs -ZMQLDFLAGS := -lzmq -L$(ZEROMQ)/src/.libs - -# Lists that the */Makefile.inc makefile fragments will add to -OBJDIRS := -TESTS := - -# Make sure that 'all' is the first target -all: - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - -# Makefile template so that the makefrags are far less redundant -MAKEFILE_TEMPLATE := Makefile.template - -# Makefile fragments for library code -include proto/Makefile.inc -include common/Makefile.inc -include backend/Makefile.inc -include paxos/Makefile.inc - -# Makefile fragments for components -include applications/Makefile.inc -include scheduler/Makefile.inc -include sequencer/Makefile.inc - -# Makefile fragments for executable and deployment scripts -include deployment/Makefile.inc - -test: $(TESTS) - -clean: - rm -rf $(OBJDIR) $(BINDIR) $(DBDIR) $(STORE) $(CHKPNT) - -# This magic automatically generates makefile dependencies -# for header files included from C source files we compile, -# and keeps those dependencies up-to-date every time we recompile. -# See 'mergedep.pl' for more information. -$(OBJDIR)/.deps: $(foreach dir, $(OBJDIRS), $(wildcard $(OBJDIR)/$(dir)/*.d)) - @mkdir -p $(@D) - @cat $^ /dev/null > $@ -# @$(PERL) mergedep.pl $@ $^ --include $(OBJDIR)/.deps - -always: - -# Eliminate default suffix rules -.SUFFIXES: - -# make it so that no intermediate .o files are ever deleted -.PRECIOUS: $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.o) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.cc) \ - $(foreach dir, $(OBJDIRS), $(OBJDIR)/$(dir)/%.pb.h) - -.PHONY: all always clean test diff --git a/src_traditional_2pl_2pc_4_partitions/Makefile.template b/src_traditional_2pl_2pc_4_partitions/Makefile.template deleted file mode 100755 index 4ace61a6..00000000 --- a/src_traditional_2pl_2pc_4_partitions/Makefile.template +++ /dev/null @@ -1,40 +0,0 @@ -# Build files only if they exist. -$(UPPERC_DIR)_SRCS := $(wildcard $($(UPPERC_DIR)_SRCS)) -$(UPPERC_DIR)_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_SRCS)) - -# Protobuf specific instructions -ifeq ($(LOWERC_DIR),proto) - $(UPPERC_DIR)_OBJS := $(patsubst %.proto, $(OBJDIR)/%.pb.o, $($(UPPERC_DIR)_OBJS)) -endif - -$(UPPERC_DIR)_TEST_SRCS := $(wildcard $(LOWERC_DIR)/*_test.cc) -$(UPPERC_DIR)_TEST_OBJS := $(patsubst %.cc, $(OBJDIR)/%.o, $($(UPPERC_DIR)_TEST_SRCS)) -$(UPPERC_DIR)_TESTS := $(patsubst %.cc, $(BINDIR)/%, $($(UPPERC_DIR)_TEST_SRCS)) - -TESTS += test-$(LOWERC_DIR) - -all: $(LOWERC_DIR)-all - -$(LOWERC_DIR)-all: $(LOWERC_DIR) $(LOWERC_DIR)-tests -$(LOWERC_DIR): $(patsubst %, $(BINDIR)/%, $($(UPPERC_DIR)_PROG))\ - $($(UPPERC_DIR)_OBJS) -$(LOWERC_DIR)-tests: $($(UPPERC_DIR)_TESTS) - -ifeq ($(LOWERC_DIR),proto) - SRC_CC_FILES := -else - SRC_CC_FILES := $(LOWERC_DIR)/%.cc -endif - -$(OBJDIR)/$(LOWERC_DIR)/%.o: $(SRC_CC_FILES) $(SRC_LINKED_OBJECTS) - @echo + cxx $< - @mkdir -p $(@D) - $(V)$(CXX) $(CXXFLAGS) -o $@ -c $< - -$(BINDIR)/$(LOWERC_DIR)/%_test: $(OBJDIR)/$(LOWERC_DIR)/%_test.o $($(UPPERC_DIR)_OBJS) \ - $(TEST_LINKED_OBJECTS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) $(ZMQLDFLAGS) - -.PHONY: $(LOWERC_DIR)-all $(LOWERC_DIR) $(LOWERC_DIR)-tests test-$(LOWERC_DIR) diff --git a/src_traditional_2pl_2pc_4_partitions/applications/.DS_Store b/src_traditional_2pl_2pc_4_partitions/applications/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/applications/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/applications/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/applications/Makefile.inc deleted file mode 100755 index db42fd1b..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += applications - -# Create template specific variables -UPPERC_DIR := APPLICATIONS -LOWERC_DIR := applications - -APPLICATIONS_SRCS := applications/tpcc.cc applications/microbenchmark.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-applications: $(APPLICATIONS_TESTS) - @for a in $(APPLICATIONS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/applications/application.h b/src_traditional_2pl_2pc_4_partitions/applications/application.h deleted file mode 100755 index 711284fa..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/application.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Application abstract class -// -// Application execution logic in the system is coded into - -#ifndef _DB_APPLICATIONS_APPLICATION_H_ -#define _DB_APPLICATIONS_APPLICATION_H_ - -#include - -#include "common/types.h" -#include "backend/simple_storage.h" - -using std::string; - -class Configuration; -class Storage; -class StorageManager; -class TxnProto; - -enum TxnStatus { - SUCCESS = 0, - FAILURE = 1, - REDO = 2, -}; - -class Application { - public: - virtual ~Application() {} - - // Load generation. - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const = 0; - - // Static method to convert a key into an int for an array - static int CheckpointID(Key key); - - // Execute a transaction's application logic given the input 'txn'. - virtual int Execute(TxnProto* txn, StorageManager* storage) const = 0; - - // Storage initialization method. - virtual void InitializeStorage(Storage* storage, Configuration* conf) const = 0; -}; - -#endif // _DB_APPLICATIONS_APPLICATION_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.cc b/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.cc deleted file mode 100755 index d6847d8c..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.cc +++ /dev/null @@ -1,181 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(scw): remove iostream, use cstdio instead - -#include "applications/microbenchmark.h" - -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/utils.h" -#include "common/configuration.h" -#include "proto/txn.pb.h" - -// #define PREFETCHING -#define COLD_CUTOFF 990000 - -// Fills '*keys' with num_keys unique ints k where -// 'key_start' <= k < 'key_limit', and k == part (mod nparts). -// Requires: key_start % nparts == 0 -void Microbenchmark::GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part) { - assert(key_start % nparts == 0); - keys->clear(); - for (int i = 0; i < num_keys; i++) { - // Find a key not already in '*keys'. - int key; - do { - key = key_start + part + - nparts * (rand() % ((key_limit - key_start)/nparts)); - } while (keys->count(key)); - keys->insert(key); - } -} - -TxnProto* Microbenchmark::InitializeTxn() { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(0); - txn->set_txn_type(INITIALIZE); - - // Nothing read, everything written. - for (int i = 0; i < kDBSize; i++) - txn->add_write_set(IntToString(i)); - - return txn; -} - -int cmp(const void *a, const void *b) { - return *(int *)a - *(int *)b; -} - -TxnProto* Microbenchmark::MicroTxnSP(int64 txn_id, int part) { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_SP); - txn->set_multipartition(false); - txn->set_txn_node(part); - - // Add one hot key to read/write set. - int hotkey = part + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey)); - - - // Insert set of kRWSetSize - 1 random cold keys from specified partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - kRWSetSize - 1, - nparts * hot_records, - nparts * kDBSize, - part); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - return txn; -} - -TxnProto* Microbenchmark::MicroTxnMP(int64 txn_id, int part1, int part2, int part3, int part4) { - assert(part1 != part2 && part3 != part1 && part3 != part2); - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(MICROTXN_MP); - txn->set_multipartition(true); - txn->set_txn_node(part1); - txn->add_txn_other_node(part2); - txn->add_txn_other_node(part3); - txn->add_txn_other_node(part4); - - // Add two hot keys to read/write set---one in each partition. - int hotkey1 = part1 + nparts * (rand() % hot_records); - int hotkey2 = part2 + nparts * (rand() % hot_records); - int hotkey3 = part3 + nparts * (rand() % hot_records); - int hotkey4 = part4 + nparts * (rand() % hot_records); - txn->add_read_write_set(IntToString(hotkey1)); - txn->add_read_write_set(IntToString(hotkey2)); - txn->add_read_write_set(IntToString(hotkey3)); - txn->add_read_write_set(IntToString(hotkey4)); - - // Insert set of kRWSetSize/2 - 1 random cold keys from each partition into - // read/write set. - set keys; - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part1); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 2, - nparts * hot_records, - nparts * kDBSize, - part2); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 1, - nparts * hot_records, - nparts * kDBSize, - part3); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - GetRandomKeys(&keys, - 1, - nparts * hot_records, - nparts * kDBSize, - part4); - for (set::iterator it = keys.begin(); it != keys.end(); ++it) - txn->add_read_write_set(IntToString(*it)); - - - return txn; -} - -// The load generator can be called externally to return a transaction proto -// containing a new type of transaction. -TxnProto* Microbenchmark::NewTxn(int64 txn_id, int txn_type, - string args, Configuration* config) const { - return NULL; -} - -int Microbenchmark::Execute(TxnProto* txn, StorageManager* storage) const { - // Read all elements of 'txn->read_set()', add one to each, write them all - // back out. - - for (int i = 0; i < kRWSetSize; i++) { - Value* val = storage->ReadObject(txn->read_write_set(i)); - - *val = IntToString(StringToInt(*val) + 1); - // For "long" transactions - int x = 1; - for(int i = 0; i < 1100; i++) { - x = x*x+1; - x = x+10; - x = x-2; - } - } - - return 0; -} - -void Microbenchmark::InitializeStorage(Storage* storage, Configuration* conf) const { - for (int i = 0; i < nparts * kDBSize; i++) { - if (conf->LookupPartition(IntToString(i)) == conf->this_node_id) { - storage->PutObject(IntToString(i), new Value(IntToString(i))); - } - } -} - diff --git a/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.h b/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.h deleted file mode 100755 index 455e0ac6..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/microbenchmark.h +++ /dev/null @@ -1,55 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A microbenchmark application that reads all elements of the read_set, does -// some trivial computation, and writes to all elements of the write_set. - -#ifndef _DB_APPLICATIONS_MICROBENCHMARK_H_ -#define _DB_APPLICATIONS_MICROBENCHMARK_H_ - -#include -#include - -#include "applications/application.h" -#include "backend/simple_storage.h" - -using std::set; -using std::string; - -class Microbenchmark : public Application { - public: - enum TxnType { - INITIALIZE = 0, - MICROTXN_SP = 1, - MICROTXN_MP = 2, - }; - - Microbenchmark(int nodecount, int hotcount) { - nparts = nodecount; - hot_records = hotcount; - } - - virtual ~Microbenchmark() {} - - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config = NULL) const; - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - - TxnProto* InitializeTxn(); - TxnProto* MicroTxnSP(int64 txn_id, int part1); - TxnProto* MicroTxnMP(int64 txn_id, int part1, int part2, int part3, int part4); - - int nparts; - int hot_records; - static const int kRWSetSize = 10; // MUST BE EVEN - static const int kDBSize = 1000000; - - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - private: - void GetRandomKeys(set* keys, int num_keys, int key_start, - int key_limit, int part); - Microbenchmark() {} -}; - -#endif // _DB_APPLICATIONS_MICROBENCHMARK_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/applications/tpcc.cc b/src_traditional_2pl_2pc_4_partitions/applications/tpcc.cc deleted file mode 100755 index 66a3b6b3..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/tpcc.cc +++ /dev/null @@ -1,1103 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#include "applications/tpcc.h" - -#include -#include - -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/utils.h" -#include "proto/tpcc.pb.h" -#include "proto/tpcc_args.pb.h" - -using std::string; - -// ---- THIS IS A HACK TO MAKE ITEMS WORK ON LOCAL MACHINE ---- // -unordered_map ItemList; -Value* TPCC::GetItem(Key key) const { return ItemList[key]; } -void TPCC::SetItem(Key key, Value* value) const { ItemList[key] = value; } - -// The load generator can be called externally to return a -// transaction proto containing a new type of transaction. -TxnProto* TPCC::NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const { - // Create the new transaction object - TxnProto* txn = new TxnProto(); - - // Set the transaction's standard attributes - txn->set_txn_id(txn_id); - txn->set_txn_type(txn_type); - txn->set_isolation_level(TxnProto::SERIALIZABLE); - txn->set_status(TxnProto::NEW); - txn->set_multipartition(false); - - // Parse out the arguments to the transaction - TPCCArgs* txn_args = new TPCCArgs(); - assert(txn_args->ParseFromString(args)); - bool mp = txn_args->multipartition(); - int remote_node; - txn->set_txn_node(config->this_node_id); - - if (mp) { - do { - remote_node = rand() % config->all_nodes.size(); - } while (config->all_nodes.size() > 1 && - remote_node == config->this_node_id); - txn->add_txn_other_node(remote_node); - } - - // Create an arg list - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Because a switch is not scoped we declare our variables outside of it - int warehouse_id, district_id, customer_id; - char warehouse_key[128], district_key[128], customer_key[128]; - int order_line_count; - bool invalid; - Value customer_value; - std::set items_used; - - // We set the read and write set based on type - switch (txn_type) { - // Initialize - case INITIALIZE: - // Finished with INITIALIZE txn creation - break; - - // New Order - case NEW_ORDER: - // First, we pick a local warehouse - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", - warehouse_id); - - txn->add_read_set(warehouse_key); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Finally, we pick a random customer - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - txn->add_read_set(customer_key); - - int order_number; - if(next_order_id_for_district.count(district_key)>0) { - order_number = next_order_id_for_district[district_key]; - next_order_id_for_district[district_key] ++; - } else { - order_number = 0; - next_order_id_for_district[district_key] = 1; - } - - // We set the length of the read and write set uniformly between 5 and 15 - order_line_count = (rand() % 11) + 5; - - // Let's choose a bad transaction 1% of the time - invalid = false; - - // Iterate through each order line - for (int i = 0; i < order_line_count; i++) { - // Set the item id (Invalid orders have the last item be -1) - int item; - do { - item = rand() % NUMBER_OF_ITEMS; - } while (items_used.count(item) > 0); - items_used.insert(item); - - if (invalid && i == order_line_count - 1) - item = -1; - - // Print the item key into a buffer - char item_key[128]; - snprintf(item_key, sizeof(item_key), "i%d", item); - - // Create an order line warehouse key (default is local) - char remote_warehouse_key[128]; - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "%s", warehouse_key); - - // We only do ~1% remote transactions - if (mp) { - txn->set_multipartition(true); - - // We loop until we actually get a remote one - int remote_warehouse_id; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), - "w%d", remote_warehouse_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != - remote_node); - } - - // Determine if we should add it to read set to avoid duplicates - bool needed = true; - for (int j = 0; j < txn->read_set_size(); j++) { - if (txn->read_set(j) == remote_warehouse_key) - needed = false; - } - if (needed) - txn->add_read_set(remote_warehouse_key); - - // Finally, we set the stock key to the read and write set - Key stock_key = string(remote_warehouse_key) + "s" + item_key; - txn->add_read_write_set(stock_key); - - // Set the quantity randomly within [1..10] - tpcc_args->add_quantities(rand() % 10 + 1); - - // Finally, we add the order line key to the write set - char order_line_key[128]; - snprintf(order_line_key, sizeof(order_line_key), "%so%dol%d", - district_key, order_number, i); - txn->add_write_set(order_line_key); - - } - - // Create a new order key to add to write set - char new_order_key[128]; - snprintf(new_order_key, sizeof(new_order_key), - "%sno%d", district_key, order_number); - txn->add_write_set(new_order_key); - - // Create an order key to add to write set - char order_key[128]; - snprintf(order_key, sizeof(order_key), "%so%d", - district_key, order_number); - txn->add_write_set(order_key); - - // Set the order line count in the args - tpcc_args->add_order_line_count(order_line_count); - tpcc_args->set_order_number(order_number); - break; - - // Payment - case PAYMENT: - // Specify an amount for the payment - tpcc_args->set_amount(rand() / (static_cast(RAND_MAX + 1.0)) * - 4999.0 + 1); - - // First, we pick a local warehouse - - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%dy", - warehouse_id); - txn->add_read_write_set(warehouse_key); - - // Next, we pick a district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%dy", - warehouse_id, district_id); - txn->add_read_write_set(district_key); - - // Add history key to write set - char history_key[128]; - snprintf(history_key, sizeof(history_key), "w%dh%ld", - warehouse_id, txn->txn_id()); - txn->add_write_set(history_key); - - // Next, we find the customer as a local one - if (WAREHOUSES_PER_NODE * config->all_nodes.size() == 1 || !mp) { - customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", - warehouse_id, district_id, customer_id); - - // If the probability is 15%, we make it a remote customer - } else { - int remote_warehouse_id; - int remote_district_id; - int remote_customer_id; - char remote_warehouse_key[40]; - do { - remote_warehouse_id = rand() % (WAREHOUSES_PER_NODE * - config->all_nodes.size()); - snprintf(remote_warehouse_key, sizeof(remote_warehouse_key), "w%d", - remote_warehouse_id); - - remote_district_id = rand() % DISTRICTS_PER_WAREHOUSE; - - remote_customer_id = rand() % CUSTOMERS_PER_DISTRICT; - snprintf(customer_key, sizeof(customer_key), "w%dd%dc%d", - remote_warehouse_id, remote_district_id, remote_customer_id); - } while (config->all_nodes.size() > 1 && - config->LookupPartition(remote_warehouse_key) != remote_node); - } - - // We only do secondary keying ~60% of the time - if (rand() / (static_cast(RAND_MAX + 1.0)) < 0.00) { - // Now that we have the object, let's create the txn arg - tpcc_args->set_last_name(customer_key); - txn->add_read_set(customer_key); - - // Otherwise just give a customer key - } else { - txn->add_read_write_set(customer_key); - } - - break; - - case ORDER_STATUS : - { - string customer_string; - string customer_latest_order; - string warehouse_string; - string district_string; - int customer_order_line_number; - - if(latest_order_id_for_customer.size() < 1) { - txn->set_txn_id(-1); - break; - } - pthread_mutex_lock(&mutex_); - customer_string = (*involed_customers)[rand() % involed_customers->size()]; - pthread_mutex_unlock(&mutex_); - - customer_latest_order = latest_order_id_for_customer[customer_string]; - warehouse_string = customer_string.substr(0,customer_string.find("d")); - district_string = customer_string.substr(0,customer_string.find("c")); - snprintf(customer_key, sizeof(customer_key), "%s", customer_string.c_str()); - snprintf(warehouse_key, sizeof(warehouse_key), "%s", warehouse_string.c_str()); - snprintf(district_key, sizeof(district_key), "%s", district_string.c_str()); - customer_order_line_number = order_line_number[customer_latest_order]; - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - txn->add_read_set(customer_key); - - snprintf(order_key, sizeof(order_key), "%s", customer_latest_order.c_str()); - txn->add_read_set(order_key); - char order_line_key[128]; - for(int i = 0; i < customer_order_line_number; i++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, i); - txn->add_read_set(order_line_key); - } - - tpcc_args->add_order_line_count(customer_order_line_number); - - break; - } - - - case STOCK_LEVEL: - { - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - - // Next, we pick a random district - district_id = rand() % DISTRICTS_PER_WAREHOUSE; - snprintf(district_key, sizeof(district_key), "w%dd%d", warehouse_id, district_id); - - if(latest_order_id_for_district.count(district_key) == 0) { - txn->set_txn_id(-1); - break; - } - - txn->add_read_set(warehouse_key); - txn->add_read_set(district_key); - int latest_order_number = latest_order_id_for_district[district_key]; - char order_line_key[128]; - char stock_key[128]; - tpcc_args->set_lastest_order_number(latest_order_number); - tpcc_args->set_threshold(rand()%10 + 10); - - for(int i = latest_order_number; (i >= 0) && (i > latest_order_number - 20); i--) { - snprintf(order_key, sizeof(order_key), "%so%d", district_key, i); - int ol_number = order_line_number[order_key]; - for(int j = 0; j < ol_number;j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - int item = item_for_order_line[order_line_key]; - if(items_used.count(item) > 0) { - continue; - } - items_used.insert(item); - txn->add_read_set(order_line_key); - snprintf(stock_key, sizeof(stock_key), "%ssi%d", warehouse_key, item); - txn->add_read_set(stock_key); - } - } - - break; - } - - case DELIVERY : - { - pthread_mutex_lock(&mutex_); - warehouse_id = (rand() % WAREHOUSES_PER_NODE) * config->all_nodes.size() + config->this_node_id; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", warehouse_id); - txn->add_read_set(warehouse_key); - - char order_line_key[128]; - int oldest_order; - - for(int i = 0; i < DISTRICTS_PER_WAREHOUSE; i++) { - snprintf(district_key, sizeof(district_key), "%sd%d", warehouse_key, i); - - if((smallest_order_id_for_district.count(district_key) == 0) || (smallest_order_id_for_district[district_key] > latest_order_id_for_district[district_key])) { - continue; - } else { - txn->add_read_set(district_key); - oldest_order = smallest_order_id_for_district[district_key]; - smallest_order_id_for_district[district_key] ++; - - snprintf(new_order_key, sizeof(new_order_key), "%sno%d", district_key, oldest_order); - txn->add_read_write_set(new_order_key); - } - - snprintf(order_key, sizeof(order_key), "%so%d", district_key, oldest_order); - - txn->add_read_write_set(order_key); - int ol_number = order_line_number[order_key]; - tpcc_args->add_order_line_count(ol_number); - - for(int j = 0; j < ol_number; j++) { - snprintf(order_line_key, sizeof(order_line_key), "%sol%d", order_key, j); - txn->add_read_write_set(order_line_key); - } - - snprintf(customer_key, sizeof(customer_key), "%s", customer_for_order[order_key].c_str()); - - txn->add_read_write_set(customer_key); - } - pthread_mutex_unlock(&mutex_); - - break; - } - - // Invalid transaction - default: - break; - } - - // Set the transaction's args field to a serialized version - Value args_string; - assert(tpcc_args->SerializeToString(&args_string)); - txn->set_arg(args_string); - - // Free memory - delete tpcc_args; - delete txn_args; - - return txn; -} - -// The execute function takes a single transaction proto and executes it based -// on what the type of the transaction is. -int TPCC::Execute(TxnProto* txn, StorageManager* storage) const { - switch (txn->txn_type()) { - // Initialize - case INITIALIZE: - InitializeStorage(storage->GetStorage(), NULL); - return SUCCESS; - break; - - // New Order - case NEW_ORDER: - return NewOrderTransaction(txn, storage); - break; - - // Payment - case PAYMENT: - return PaymentTransaction(txn, storage); - break; - - case ORDER_STATUS: - return OrderStatusTransaction(txn, storage); - break; - - case STOCK_LEVEL: - return StockLevelTransaction(txn, storage); - break; - - case DELIVERY: - return DeliveryTransaction(txn, storage); - break; - - // Invalid transaction - default: - return FAILURE; - break; - } - - return FAILURE; -} - -// The new order function is executed when the application receives a new order -// transaction. This follows the TPC-C standard. -int TPCC::NewOrderTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we retrieve the warehouse from storage - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we retrieve the district - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - // Increment the district's next order ID and put to datastore - district->set_next_order_id(district->next_order_id() + 1); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district->id(), district_value); - - // Retrieve the customer we are looking for - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(1)); - assert(customer->ParseFromString(*customer_value)); - - // Next, we get the order line count, system time, and other args from the - // transaction proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - int order_number = tpcc_args->order_number(); - double system_time = tpcc_args->system_time(); - - // Next we create an Order object - Key order_key = txn->write_set(order_line_count + 1); - Order* order = new Order(); - order->set_id(order_key); - order->set_warehouse_id(warehouse->id()); - order->set_district_id(district->id()); - order->set_customer_id(customer->id()); - - // Set some of the auxiliary data - order->set_entry_date(system_time); - order->set_carrier_id(-1); - order->set_order_line_count(order_line_count); - order->set_all_items_local(txn->multipartition()); - - // We initialize the order line amount total to 0 - int order_line_amount_total = 0; - - for (int i = 0; i < order_line_count; i++) { - // For each order line we parse out the three args - - string stock_key = txn->read_write_set(i + 1); - string supply_warehouse_key = stock_key.substr(0, stock_key.find("s")); - int quantity = tpcc_args->quantities(i); - - // Find the item key within the stock key - size_t item_idx = stock_key.find("i"); - string item_key = stock_key.substr(item_idx, string::npos); - - // First, we check if the item number is valid - Item* item = new Item(); - if (item_key == "i-1") - return FAILURE; - else - assert(item->ParseFromString(*GetItem(item_key))); - - // Next, we create a new order line object with std attributes - OrderLine* order_line = new OrderLine(); - Key order_line_key = txn->write_set(i); - order_line->set_order_id(order_line_key); - - // Set the attributes for this order line - order_line->set_district_id(district->id()); - order_line->set_warehouse_id(warehouse->id()); - order_line->set_number(i); - order_line->set_item_id(item_key); - order_line->set_supply_warehouse_id(supply_warehouse_key); - order_line->set_quantity(quantity); - order_line->set_delivery_date(system_time); - - // Next, we get the correct stock from the data store - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(stock_key); - assert(stock->ParseFromString(*stock_value)); - - // Once we have it we can increase the YTD, order_count, and remote_count - stock->set_year_to_date(stock->year_to_date() + quantity); - stock->set_order_count(stock->order_count() - 1); - if (txn->multipartition()) - stock->set_remote_count(stock->remote_count() + 1); - - // And we decrease the stock's supply appropriately and rewrite to storage - if (stock->quantity() >= quantity + 10) - stock->set_quantity(stock->quantity() - quantity); - else - stock->set_quantity(stock->quantity() - quantity + 91); - - // Put the stock back into the database - assert(stock->SerializeToString(stock_value)); - // Not necessary since storage already has a ptr to stock_value. - // storage->PutObject(stock_key, stock_value); - delete stock; - - // Next, we update the order line's amount and add it to the running sum - order_line->set_amount(quantity * item->price()); - order_line_amount_total += (quantity * item->price()); - - // Finally, we write the order line to storage - Value* order_line_value = new Value(); - assert(order_line->SerializeToString(order_line_value)); - storage->PutObject(order_line_key, order_line_value); - //order->add_order_line_ptr(reinterpret_cast(order_line_value)); - - pthread_mutex_lock(&mutex_for_item); - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) - item_for_order_line[order_line_key] = StringToInt(item_key); - pthread_mutex_unlock(&mutex_for_item); - // Free memory - delete order_line; - delete item; - } - - // We create a new NewOrder object - Key new_order_key = txn->write_set(order_line_count); - NewOrder* new_order = new NewOrder(); - new_order->set_id(new_order_key); - new_order->set_warehouse_id(warehouse->id()); - new_order->set_district_id(district->id()); - - // Serialize it and put it in the datastore - Value* new_order_value = new Value(); - assert(new_order->SerializeToString(new_order_value)); - storage->PutObject(new_order_key, new_order_value); - - // Serialize order and put it in the datastore - Value* order_value = new Value(); - assert(order->SerializeToString(order_value)); - storage->PutObject(order_key, order_value); - - if (storage->configuration_->this_node_id == storage->configuration_->LookupPartition(txn->read_set(0))) { - pthread_mutex_lock(&mutex_); - if(latest_order_id_for_customer.count(txn->read_set(1)) == 0) - involed_customers->push_back(txn->read_set(1)); - latest_order_id_for_customer[txn->read_set(1)] = order_key; - - if(smallest_order_id_for_district.count(txn->read_write_set(0)) == 0) { - smallest_order_id_for_district[txn->read_write_set(0)] = order_number; - } - order_line_number[order_key] = order_line_count; - customer_for_order[order_key] = txn->read_set(1); - latest_order_id_for_district[txn->read_write_set(0)] = order_number; - pthread_mutex_unlock(&mutex_); - } - - // Successfully completed transaction - delete warehouse; - delete district; - delete customer; - delete order; - delete new_order; - delete tpcc_args; - return SUCCESS; -} - -// The payment function is executed when the application receives a -// payment transaction. This follows the TPC-C standard. -int TPCC::PaymentTransaction(TxnProto* txn, StorageManager* storage) const { - // First, we parse out the transaction args from the TPCC proto - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int amount = tpcc_args->amount(); - - // We create a string to hold up the customer object we look up - Value* customer_value; - Key customer_key; - - // If there's a last name we do secondary keying - if (tpcc_args->has_last_name()) { - Key secondary_key = tpcc_args->last_name(); - - // If the RW set is not at least of size 3, then no customer key was - // given to this transaction. Otherwise, we perform a check to ensure - // the secondary key we just looked up agrees with our previous lookup - if (txn->read_write_set_size() < 3 || secondary_key != txn->read_write_set(2)) { - // Append the newly read key to write set - if (txn->read_write_set_size() < 3) - txn->add_read_write_set(secondary_key); - - // Or the old one was incorrect so we overwrite it - else - txn->set_read_write_set(2, secondary_key); - - return REDO; - - // Otherwise, we look up the customer's key - } else { - customer_value = storage->ReadObject(tpcc_args->last_name()); - } - - // Otherwise we use the final argument - } else { - customer_key = txn->read_write_set(2); - customer_value = storage->ReadObject(customer_key); - } - - // Deserialize the warehouse object - Key warehouse_key = txn->read_write_set(0); - Value* warehouse_value = storage->ReadObject(warehouse_key); - Warehouse* warehouse = new Warehouse(); - assert(warehouse->ParseFromString(*warehouse_value)); - - // Next, we update the values of the warehouse and write it out - warehouse->set_year_to_date(warehouse->year_to_date() + amount); - assert(warehouse->SerializeToString(warehouse_value)); - // Not necessary since storage already has a pointer to warehouse_value. - // storage->PutObject(warehouse_key, warehouse_value); - - // Deserialize the district object - Key district_key = txn->read_write_set(1); - Value* district_value = storage->ReadObject(district_key); - District* district = new District(); - assert(district->ParseFromString(*district_value)); - - // Next, we update the values of the district and write it out - district->set_year_to_date(district->year_to_date() + amount); - assert(district->SerializeToString(district_value)); - // Not necessary since storage already has a pointer to district_value. - // storage->PutObject(district_key, district_value); - - // We deserialize the customer - Customer* customer = new Customer(); - assert(customer->ParseFromString(*customer_value)); - - // Next, we update the customer's balance, payment and payment count - customer->set_balance(customer->balance() - amount); - customer->set_year_to_date_payment(customer->year_to_date_payment() + amount); - customer->set_payment_count(customer->payment_count() + 1); - - // If the customer has bad credit, we update the data information attached - // to her - if (customer->credit() == "BC") { - string data = customer->data(); - char new_information[500]; - - // Print the new_information into the buffer - snprintf(new_information, sizeof(new_information), "%s%s%s%s%s%d%s", - customer->id().c_str(), customer->warehouse_id().c_str(), - customer->district_id().c_str(), district->id().c_str(), - warehouse->id().c_str(), amount, customer->data().c_str()); - } - - // We write the customer to disk - assert(customer->SerializeToString(customer_value)); - // Not necessary since storage already has a pointer to customer_value. - // storage->PutObject(customer_key, customer_value); - - // Finally, we create a history object and update the data - History* history = new History(); - history->set_customer_id(customer_key); - history->set_customer_warehouse_id(customer->warehouse_id()); - history->set_customer_district_id(customer->district_id()); - history->set_warehouse_id(warehouse_key); - history->set_district_id(district_key); - - // Create the data for the history object - char history_data[100]; - snprintf(history_data, sizeof(history_data), "%s %s", warehouse->name().c_str(), district->name().c_str()); - history->set_data(history_data); - - // Write the history object to disk - Value* history_value = new Value(); - assert(history->SerializeToString(history_value)); - storage->PutObject(txn->write_set(0), history_value); - - // Successfully completed transaction - delete customer; - delete history; - delete district; - delete warehouse; - delete tpcc_args; - return SUCCESS; -} - - -int TPCC::OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int order_line_count = tpcc_args->order_line_count(0); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - Value* customer_value; - Customer* customer = new Customer(); - customer_value = storage->ReadObject(txn->read_set(2)); - assert(customer->ParseFromString(*customer_value)); - - string customer_first = customer->first(); - string customer_middle = customer->middle(); - string customer_last = customer->last(); - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_set(3)); - - assert(order->ParseFromString(*order_value)); - - for(int i = 0; i < order_line_count; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(4+i)); - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - string supply_warehouse_id = order_line->supply_warehouse_id(); - - delete order_line; - } - - delete warehouse; - delete district; - delete customer; - delete order; - return SUCCESS; -} - -int TPCC::StockLevelTransaction(TxnProto* txn, StorageManager* storage) const { - int low_stock = 0; - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - int threshold = tpcc_args->threshold(); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(1)); - assert(district->ParseFromString(*district_value)); - - int index = 0; - - int cycle = (txn->read_set_size() - 2)/2; - for(int i = 0; i < cycle; i++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(order_line->ParseFromString(*order_line_value)); - string item_key = order_line->item_id(); - - Stock* stock = new Stock(); - Value* stock_value = storage->ReadObject(txn->read_set(2+index)); - index ++; - assert(stock->ParseFromString(*stock_value)); - if(stock->quantity() < threshold) { - low_stock ++; - } - delete order_line; - delete stock; - } - - delete warehouse; - delete district; - return SUCCESS; -} - -int TPCC::DeliveryTransaction(TxnProto* txn, StorageManager* storage) const { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->ParseFromString(txn->arg()); - - Value* warehouse_value; - Warehouse* warehouse = new Warehouse(); - warehouse_value = storage->ReadObject(txn->read_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - - if(txn->read_set_size() == 1) { - delete warehouse; - return SUCCESS; - } - - int delivery_district_number = txn->read_set_size() - 1; - int read_write_index = 0; - int line_count_index = 0; - for(int i = 1; i <= delivery_district_number; i++) { - District* district = new District(); - Value* district_value = storage->ReadObject(txn->read_set(i)); - assert(district->ParseFromString(*district_value)); - - storage->DeleteObject(txn->read_write_set(read_write_index)); - read_write_index ++; - - Order* order = new Order(); - Value* order_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order->ParseFromString(*order_value)); - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "%s", order->customer_id().c_str()); - - order->set_carrier_id(rand()%10); - - int ol_number = tpcc_args->order_line_count(line_count_index); - line_count_index ++; - double total_amount = 0; - - for(int j = 0; j < ol_number; j++) { - OrderLine* order_line = new OrderLine(); - Value* order_line_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(order_line->ParseFromString(*order_line_value)); - order_line->set_delivery_date(GetTime()); - total_amount = total_amount + order_line->amount(); - - delete order_line; - } - - - Customer* customer = new Customer(); - Value* customer_value = storage->ReadObject(txn->read_write_set(read_write_index)); - read_write_index ++; - assert(customer->ParseFromString(*customer_value)); - customer->set_balance(customer->balance() + total_amount); - customer->set_delivery_count(customer->delivery_count() + 1); - - delete district; - delete order; - delete customer; - - } - - delete warehouse; - return SUCCESS; -} - - -// The initialize function is executed when an initialize transaction comes -// through, indicating we should populate the database with fake data -void TPCC::InitializeStorage(Storage* storage, Configuration* conf) const { - // We create and write out all of the warehouses - for (int i = 0; i < (int)(WAREHOUSES_PER_NODE * conf->all_nodes.size()); i++) { - // First, we create a key for the warehouse - char warehouse_key[128], warehouse_key_ytd[128]; - Value* warehouse_value = new Value(); - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - snprintf(warehouse_key_ytd, sizeof(warehouse_key_ytd), "w%dy", i); - if (conf->LookupPartition(warehouse_key) != conf->this_node_id) { - continue; - } - // Next we initialize the object and serialize it - Warehouse* warehouse = CreateWarehouse(warehouse_key); - assert(warehouse->SerializeToString(warehouse_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(warehouse_key) == conf->this_node_id) { - storage->PutObject(warehouse_key, warehouse_value); - storage->PutObject(warehouse_key_ytd, new Value(*warehouse_value)); - } - - // Next, we create and write out all of the districts - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - // First, we create a key for the district - char district_key[128], district_key_ytd[128]; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - snprintf(district_key_ytd, sizeof(district_key_ytd), "w%dd%dy", - i, j); - - // Next we initialize the object and serialize it - Value* district_value = new Value(); - District* district = CreateDistrict(district_key, warehouse_key); - assert(district->SerializeToString(district_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(district_key) == conf->this_node_id) { - storage->PutObject(district_key, district_value); - storage->PutObject(district_key_ytd, new Value(*district_value)); - } - - // Next, we create and write out all of the customers - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - // First, we create a key for the customer - char customer_key[128]; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - - // Next we initialize the object and serialize it - Value* customer_value = new Value(); - Customer* customer = CreateCustomer(customer_key, district_key, - warehouse_key); - assert(customer->SerializeToString(customer_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(customer_key) == conf->this_node_id) - storage->PutObject(customer_key, customer_value); - delete customer; - } - - // Free storage - delete district; - } - - // Next, we create and write out all of the stock - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - // First, we create a key for the stock - char item_key[128]; - Value* stock_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", j); - - // Next we initialize the object and serialize it - Stock* stock = CreateStock(item_key, warehouse_key); - assert(stock->SerializeToString(stock_value)); - - // Finally, we pass it off to the storage manager to write to disk - if (conf->LookupPartition(stock->id()) == conf->this_node_id) - storage->PutObject(stock->id(), stock_value); - delete stock; - } - - // Free storage - delete warehouse; - } - - // Finally, all the items are initialized - srand(1000); - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - // First, we create a key for the item - char item_key[128]; - Value* item_value = new Value(); - snprintf(item_key, sizeof(item_key), "i%d", i); - - // Next we initialize the object and serialize it - Item* item = CreateItem(item_key); - assert(item->SerializeToString(item_value)); - - // Finally, we pass it off to the local record of items - SetItem(string(item_key), item_value); - delete item; - } -} - -// The following method is a dumb constructor for the warehouse protobuffer -Warehouse* TPCC::CreateWarehouse(Key warehouse_key) const { - Warehouse* warehouse = new Warehouse(); - - // We initialize the id and the name fields - warehouse->set_id(warehouse_key); - warehouse->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - warehouse->set_street_1(RandomString(20)); - warehouse->set_street_2(RandomString(20)); - warehouse->set_city(RandomString(20)); - warehouse->set_state(RandomString(2)); - warehouse->set_zip(RandomString(9)); - - // Set default financial information - warehouse->set_tax(0.05); - warehouse->set_year_to_date(0.0); - - return warehouse; -} - -District* TPCC::CreateDistrict(Key district_key, Key warehouse_key) const { - District* district = new District(); - - // We initialize the id and the name fields - district->set_id(district_key); - district->set_warehouse_id(warehouse_key); - district->set_name(RandomString(10)); - - // Provide some information to make TPC-C happy - district->set_street_1(RandomString(20)); - district->set_street_2(RandomString(20)); - district->set_city(RandomString(20)); - district->set_state(RandomString(2)); - district->set_zip(RandomString(9)); - - // Set default financial information - district->set_tax(0.05); - district->set_year_to_date(0.0); - district->set_next_order_id(1); - - return district; -} - -Customer* TPCC::CreateCustomer(Key customer_key, Key district_key, - Key warehouse_key) const { - Customer* customer = new Customer(); - - // We initialize the various keys - customer->set_id(customer_key); - customer->set_district_id(district_key); - customer->set_warehouse_id(warehouse_key); - - // Next, we create a first and middle name - customer->set_first(RandomString(20)); - customer->set_middle(RandomString(20)); - customer->set_last(customer_key); - - // Provide some information to make TPC-C happy - customer->set_street_1(RandomString(20)); - customer->set_street_2(RandomString(20)); - customer->set_city(RandomString(20)); - customer->set_state(RandomString(2)); - customer->set_zip(RandomString(9)); - - // Set default financial information - customer->set_since(0); - customer->set_credit("GC"); - customer->set_credit_limit(0.01); - customer->set_discount(0.5); - customer->set_balance(0); - customer->set_year_to_date_payment(0); - customer->set_payment_count(0); - customer->set_delivery_count(0); - - // Set some miscellaneous data - customer->set_data(RandomString(50)); - - return customer; -} - -Stock* TPCC::CreateStock(Key item_key, Key warehouse_key) const { - Stock* stock = new Stock(); - - // We initialize the various keys - char stock_key[128]; - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key.c_str(), item_key.c_str()); - stock->set_id(stock_key); - stock->set_warehouse_id(warehouse_key); - stock->set_item_id(item_key); - - // Next, we create a first and middle name - stock->set_quantity(rand() % 100 + 100); - - // Set default financial information - stock->set_year_to_date(0); - stock->set_order_count(0); - stock->set_remote_count(0); - - // Set some miscellaneous data - stock->set_data(RandomString(50)); - - return stock; -} - -Item* TPCC::CreateItem(Key item_key) const { - Item* item = new Item(); - - // We initialize the item's key - item->set_id(item_key); - - // Initialize some fake data for the name, price and data - item->set_name(RandomString(24)); - item->set_price(rand() % 100); - item->set_data(RandomString(50)); - - return item; -} diff --git a/src_traditional_2pl_2pc_4_partitions/applications/tpcc.h b/src_traditional_2pl_2pc_4_partitions/applications/tpcc.h deleted file mode 100755 index d6023c07..00000000 --- a/src_traditional_2pl_2pc_4_partitions/applications/tpcc.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// A concrete implementation of TPC-C (application subclass) - -#ifndef _DB_APPLICATIONS_TPCC_H_ -#define _DB_APPLICATIONS_TPCC_H_ - -#include - -#include "applications/application.h" -#include "proto/txn.pb.h" -#include "common/configuration.h" - -#define WAREHOUSES_PER_NODE 20 -#define DISTRICTS_PER_WAREHOUSE 10 -#define DISTRICTS_PER_NODE (WAREHOUSES_PER_NODE * DISTRICTS_PER_WAREHOUSE) -#define CUSTOMERS_PER_DISTRICT 3000 -#define CUSTOMERS_PER_NODE (DISTRICTS_PER_NODE * CUSTOMERS_PER_DISTRICT) -#define NUMBER_OF_ITEMS 100000 - -using std::string; - -class Warehouse; -class District; -class Customer; -class Item; -class Stock; - -class TPCC : public Application { - public: - enum TxnType { - INITIALIZE = 0, - NEW_ORDER = 1, - PAYMENT = 2, - ORDER_STATUS = 3, - DELIVERY = 4, - STOCK_LEVEL = 5, - }; - - virtual ~TPCC() {} - - // Load generator for a new transaction - virtual TxnProto* NewTxn(int64 txn_id, int txn_type, string args, - Configuration* config) const; - - // The key converter takes a valid key (string) and converts it to an id - // for the checkpoint to use - static int CheckpointID(Key key) { - // Initial dissection of the key - size_t id_idx; - - // Switch based on key type - size_t bad = string::npos; - if ((id_idx = key.find("s")) != bad) { - size_t ware = key.find("w"); - return 1000000 + NUMBER_OF_ITEMS * atoi(&key[ware + 1]) + - atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("c")) != bad) { - return WAREHOUSES_PER_NODE + DISTRICTS_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("d")) != bad && key.find("y") == bad) { - return WAREHOUSES_PER_NODE + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("w")) != bad) { - return atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("i")) != bad) { - return 3000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ol")) != bad) { - return 4000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("no")) != bad) { - return 5000000 + atoi(&key[id_idx + 2]); - } else if ((id_idx = key.find("o")) != bad) { - return 6000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("h")) != bad) { - return 7000000 + atoi(&key[id_idx + 1]); - } else if ((id_idx = key.find("ln")) != bad) { - return 8000000 + atoi(&key[id_idx + 2]); - } - - // Invalid key - return -1; - } - - // Simple execution of a transaction using a given storage - virtual int Execute(TxnProto* txn, StorageManager* storage) const; - -/* TODO(Thad): Uncomment once testing friend class exists - private: */ - // When the first transaction is called, the following function initializes - // a set of fake data for use in the application - virtual void InitializeStorage(Storage* storage, Configuration* conf) const; - - // The following methods are simple randomized initializers that provide us - // fake data for our TPC-C function - Warehouse* CreateWarehouse(Key id) const; - District* CreateDistrict(Key id, Key warehouse_id) const; - Customer* CreateCustomer(Key id, Key district_id, Key warehouse_id) const; - Item* CreateItem(Key id) const; - Stock* CreateStock(Key id, Key warehouse_id) const; - - // A NewOrder call takes a set of args and a transaction id and performs - // the new order transaction as specified by TPC-C. The return is 1 for - // success or 0 for failure. - int NewOrderTransaction(TxnProto* txn, StorageManager* storage) const; - - // A Payment call takes a set of args as the parameter and performs the - // payment transaction, returning a 1 for success or 0 for failure. - int PaymentTransaction(TxnProto* txn, StorageManager* storage) const; - - int OrderStatusTransaction(TxnProto* txn, StorageManager* storage) const; - - int StockLevelTransaction(TxnProto* txn, StorageManager* storage) const; - - int DeliveryTransaction(TxnProto* txn, StorageManager* storage) const; - - // The following are implementations of retrieval and writing for local items - Value* GetItem(Key key) const; - void SetItem(Key key, Value* value) const; -}; - -#endif // _DB_APPLICATIONS_TPCC_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/backend/.DS_Store b/src_traditional_2pl_2pc_4_partitions/backend/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/backend/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/backend/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/backend/Makefile.inc deleted file mode 100755 index bbd628f7..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/Makefile.inc +++ /dev/null @@ -1,25 +0,0 @@ -OBJDIRS += backend - -# Create template specific variables -UPPERC_DIR := BACKEND -LOWERC_DIR := backend - -BACKEND_SRCS := backend/simple_storage.cc \ - backend/storage_manager.cc - -SRC_LINKED_OBJECTS := $(PROTO_OBJS) -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(CHECKPOINT_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-backend: $(BACKEND_TESTS) - @echo + Ensuring Database Storage Layer Exists... - @mkdir -p $(STORE) - @echo + Ensuring Checkpoint Storage Layer Exists... - @mkdir -p $(CHKPNT) - @for a in $(BACKEND_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.cc b/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.cc deleted file mode 100755 index aa0f99f9..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is the implementation for a versioned database backend - -#include "backend/collapsed_versioned_storage.h" - -#include -#include -#include - -using std::string; - -#define TPCCHACK - -#ifdef TPCCHACK -#define MAXARRAYSIZE 1000000 - // For inserted objects we need to make a representation that is thread-safe - // (i.e. an array). This is kind of hacky, but since this only corresponds - // to TPCC, we'll be okay - Value* NewOrderStore[MAXARRAYSIZE]; - Value* OrderStore[MAXARRAYSIZE]; - Value* OrderLineStore[MAXARRAYSIZE * 15]; - Value* HistoryStore[MAXARRAYSIZE]; -#endif - -Value* CollapsedVersionedStorage::ReadObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - return OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])]; - } else if (key.find("no") != string::npos) { - return NewOrderStore[txn_id]; - } else if (key.find("o") != string::npos) { - return OrderStore[txn_id]; - } else if (key.find("h") != string::npos) { - return HistoryStore[txn_id]; - } else { -#endif - - // Check to see if a match even exists - if (objects_.count(key) != 0) { - for (DataNode* list = objects_[key]; list; list = list->next) { - if (list->txn_id <= txn_id) { - return list->value; - } - } - } - -#ifdef TPCCHACK - } -#endif - - // No match found - return NULL; -} - -bool CollapsedVersionedStorage::PutObject(const Key& key, Value* value, - int64 txn_id) { -#ifdef TPCCHACK - if (key.find("ol") != string::npos) { - OrderLineStore[txn_id * 15 + atoi(&key[key.find("line(") + 5])] = - value; - } else if (key.find("no") != string::npos) { - NewOrderStore[txn_id] = value; - } else if (key.find("o") != string::npos) { - OrderStore[txn_id] = value; - } else if (key.find("h") != string::npos) { - HistoryStore[txn_id] = value; - } else { -#endif - - // Create the new version to insert into the list - DataNode* item = new DataNode(); - item->txn_id = txn_id; - item->value = value; - item->next = NULL; - - // Is the most recent value a candidate for pruning? - DataNode* current; - if (objects_.count(key) != 0 && (current = objects_[key]) != NULL) { - int64 most_recent = current->txn_id; - - if ((most_recent > stable_ && txn_id > stable_) || - (most_recent <= stable_ && txn_id <= stable_)) { - item->next = current->next; - delete current; - } else { - item->next = current; - } - } - objects_[key] = item; - -#ifdef TPCCHACK - } -#endif - - return true; -} - -bool CollapsedVersionedStorage::DeleteObject(const Key& key, int64 txn_id) { -#ifdef TPCCHACK - if (key.find("o") != string::npos || key.find("h") != string::npos) - return false; -#endif - - DataNode* list = (objects_.count(key) == 0 ? NULL : objects_[key]); - - while (list != NULL) { - if ((list->txn_id > stable_ && txn_id > stable_) || - (list->txn_id <= stable_ && txn_id <= stable_)) - break; - - list = list->next; - } - - // First we need to insert an empty string when there is >1 item - if (list != NULL && objects_[key] == list && list->next != NULL) { - objects_[key]->txn_id = txn_id; - objects_[key]->value = NULL; - - // Otherwise we need to free the head - } else if (list != NULL && objects_[key] == list) { - delete objects_[key]; - objects_[key] = NULL; - - // Lastly, we may only want to free the tail - } else if (list != NULL) { - delete list; - objects_[key]->next = NULL; - } - - return true; -} - -int CollapsedVersionedStorage::Checkpoint() { - pthread_t checkpointing_daemon; - int thread_status = pthread_create(&checkpointing_daemon, NULL, - &RunCheckpointer, this); - - return thread_status; -} - -void CollapsedVersionedStorage::CaptureCheckpoint() { - // Give the user output - fprintf(stdout, "Beginning checkpoint capture...\n"); - - // First, we open the file for writing - char log_name[200]; - snprintf(log_name, sizeof(log_name), "%s/%ld.checkpoint", CHKPNTDIR, stable_); - FILE* checkpoint = fopen(log_name, "w"); - - // Next we iterate through all of the objects and write the stable version - // to disk - unordered_map::iterator it; - for (it = objects_.begin(); it != objects_.end(); it++) { - // Read in the stable value - Key key = it->first; - Value* result = ReadObject(key, stable_); - - // Write to disk - int key_length = key.length(); - int val_length = result->length(); - fprintf(checkpoint, "%c%c%c%c%s%c%c%c%c%s", - static_cast(key_length >> 24), - static_cast(key_length >> 16), - static_cast(key_length >> 8), - static_cast(key_length), - key.c_str(), - static_cast(val_length >> 24), - static_cast(val_length >> 16), - static_cast(val_length >> 8), - static_cast(val_length), - result->c_str()); - - // Remove object from tree if there's an old version - if (it->second->next != NULL) - DeleteObject(key, stable_); - } - -#ifdef TPCCHACK - fprintf(checkpoint, "\nNewOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (NewOrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*NewOrderStore[i]).length()), - (*NewOrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrder\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (OrderStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderStore[i]).length()), - (*OrderStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nOrderLine\n"); - for (int64 i = 0; i < MAXARRAYSIZE * 15; i++) { - if (OrderLineStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*OrderLineStore[i]).length()), - (*OrderLineStore[i]).c_str()); - } - } - - fprintf(checkpoint, "\nHistory\n"); - for (int64 i = 0; i < MAXARRAYSIZE; i++) { - if (HistoryStore[i] != NULL) { - fprintf(checkpoint, "%ld%s", - static_cast((*HistoryStore[i]).length()), - (*HistoryStore[i]).c_str()); - } - } -#endif - - // Close the file - fclose(checkpoint); - - // Give the user output - fprintf(stdout, "Finished checkpointing\n"); -} diff --git a/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.h b/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.h deleted file mode 100755 index 249a40d6..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/collapsed_versioned_storage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This implements a simple collapsed storage that can be used in a versioned -// deterministic database system. - -#ifndef _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ - -#include -#include -#include - -#include "backend/versioned_storage.h" - -#define CHKPNTDIR "../db/checkpoints" - -using std::tr1::unordered_map; - -struct DataNode { - int64 txn_id; - Value* value; - DataNode* next; -}; - -class CollapsedVersionedStorage : public VersionedStorage { - public: - CollapsedVersionedStorage() { - stable_ = 0; - -#ifdef TPCCHACK - memset(&NewOrderStore, 0, sizeof(NewOrderStore)); - memset(&OrderStore, 0, sizeof(OrderStore)); - memset(&OrderLineStore, 0, sizeof(OrderLineStore)); - memset(&HistoryStore, 0, sizeof(HistoryStore)); -#endif - } - virtual ~CollapsedVersionedStorage() {} - - // TODO(Thad and Philip): How can we incorporate this type of versioned - // storage into the work that you've been doing with prefetching? It seems - // like we could do something optimistic with writing to disk and avoiding - // having to checkpoint, but we should see. - - // Standard operators in the DB - virtual Value* ReadObject(const Key& key, int64 txn_id = LLONG_MAX); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id); - virtual bool DeleteObject(const Key& key, int64 txn_id); - - // Specify the overloaded parent functions we are using here - using VersionedStorage::Prefetch; - using VersionedStorage::Unfetch; - - // At a new versioned state, the version system is notified that the - // previously stable values are no longer necessary. At this point in time, - // the database can switch the labels as to what is stable (the previously - // frozen values) to a new txn_id occurring in the future. - virtual void PrepareForCheckpoint(int64 stable) { stable_ = stable; } - virtual int Checkpoint(); - - // The capture checkpoint method is an internal method that allows us to - // write out the stable checkpoint to disk. - virtual void CaptureCheckpoint(); - - private: - // We make a simple mapping of keys to a map of "versions" of our value. - // The int64 represents a simple transaction id and the Value associated with - // it is whatever value was written out at that time. - unordered_map objects_; - - // The stable and frozen int64 represent which transaction ID's are stable - // to write out to storage, and which should be the latest to be overwritten - // in the current database execution cycle, respectively. - int64 stable_; -}; - -static inline void* RunCheckpointer(void* storage) { - (reinterpret_cast(storage))->CaptureCheckpoint(); - return NULL; -} - -#endif // _DB_BACKEND_COLLAPSED_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.cc b/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.cc deleted file mode 100755 index f5514372..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#include "backend/fetching_storage.h" - -typedef FetchingStorage::Latch Latch; - -////////////////// Constructors/Destructors ////////////////////// - -// Singleton constructor - -FetchingStorage* FetchingStorage::self = NULL; - -FetchingStorage* FetchingStorage::BuildStorage() { - if (self == NULL) - self = new FetchingStorage(); - return self; -} - -// Private constructor - -FetchingStorage::FetchingStorage() { - main_memory_ = new SimpleStorage(); - // 1 MILLION LATCHES! - latches_ = new Latch[1000000]; - - pthread_create(&gc_thread_, NULL, RunGCThread, - reinterpret_cast(this)); -} - -FetchingStorage::~FetchingStorage() { - delete main_memory_; - delete[] latches_; -} - -////////////////// Private utility functions ////////////////////// - -Latch* FetchingStorage::LatchFor(const Key& key) { - // Just fail miserably if we are passed a non-int. - // assert(atoi(key.c_str()) != 0); - // An array is a nice threadsafe hashtable. - return latches_ + atoi(key.c_str()); -} - -void FetchingStorage::GetKey(int fd, Key* key) { - char path[255]; - snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); - char key_c_str[255]; - memset(&key_c_str, 0, 255); - readlink(path, key_c_str, 255); - *key = string((strrchr(key_c_str, '/') + 1)); -} - -void* FetchingStorage::RunGCThread(void *arg) { - FetchingStorage* storage = reinterpret_cast(arg); - while (true) { - double start_time = GetTime(); - for (int i = COLD_CUTOFF; i < 1000000; i++) { - storage->HardUnfetch(IntToString(i)); - } - usleep(static_cast(1000000*(GetTime()-start_time))); - } - return NULL; -} - -///////////// The meat and potato public interface methods. /////////// - -Value* FetchingStorage::ReadObject(const Key& key, int64 txn_id) { - Latch* latch = LatchFor(key); - // Must call a Prefetch before transaction. - pthread_mutex_lock(&latch->lock_); - assert(latch->state != ON_DISK); - assert(latch->state != RELEASING); - assert(latch->state == FETCHING || latch->state == IN_MEMORY); - assert(latch->active_requests > 0); - pthread_mutex_unlock(&latch->lock_); - // Block thread until pre-fetch on this key is done. - while (latch->state == FETCHING) {} - return main_memory_->ReadObject(key); -} - -// Write data to memory. -bool FetchingStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Must call a prefetch before transaction - assert(latch->active_requests > 0); - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - main_memory_->PutObject(key, value); - return true; -} - -// Put null, change state to uninitialized. -bool FetchingStorage::DeleteObject(const Key& key, int64 txn_id) { - return PutObject(key, NULL, txn_id); -} - -// Return false if file does not exist. -bool FetchingStorage::Prefetch(const Key& key, double* wait_time) { - Latch* latch = LatchFor(key); - - pthread_mutex_lock(&latch->lock_); - - int active_requests = latch->active_requests; - latch->active_requests++; - - State previous_state = latch->state; - if (previous_state == ON_DISK) - latch->state = FETCHING; - if (previous_state == UNINITIALIZED) { - main_memory_->PutObject(key, new Value()); - latch->state = IN_MEMORY; - } - if (previous_state == RELEASING) - latch->state = IN_MEMORY; - State current_state = latch->state; - - pthread_mutex_unlock(&latch->lock_); - - // Pre-fetch and in memory pre-states are no-ops. - if (current_state == IN_MEMORY) { - *wait_time = 0; // You're good to go. - return true; - } else if (previous_state == FETCHING) { - // We already have another prefetching attempt. - *wait_time = 0.100; // arbitrary nonzero result. - return true; - } else { - // Not in memory: cold call to prefetch. - assert(active_requests == 0); - *wait_time = 0.100; // somewhat larger arbitrary nonzero result. - char* buf = new char[PAGE_SIZE]; - return FileRead(key, buf, PAGE_SIZE); - } -} - -bool FetchingStorage::HardUnfetch(const Key& key) { - // Since we have a write lock, we know there are no concurrent - // reads to this key, so we can freely read as well. - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - - State previous_state = latch->state; - int active_requests = latch->active_requests; - - // Only one of the following two conditions can be true. - if (active_requests == 0) - latch->state = RELEASING; - if (active_requests == 0 && latch->state == FETCHING) - latch->state = ON_DISK; - - pthread_mutex_unlock(&latch->lock_); - - if (active_requests == 0 && previous_state == IN_MEMORY) { - Value* result = main_memory_->ReadObject(key); - int len = strlen(result->c_str()); - char* c_result = new char[len+1]; - strcpy(c_result, result->c_str()); - return FilePut(key, c_result, len); - } else { - return true; - } -} - -bool FetchingStorage::Unfetch(const Key& key) { - Latch* latch = LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State state = latch->state; - latch->active_requests--; - assert(latch->active_requests >= 0); - assert(latch->state == FETCHING || latch->state == RELEASING || - latch->state == IN_MEMORY); - pthread_mutex_unlock(&latch->lock_); - if (state == UNINITIALIZED) - HardUnfetch(key); - return true; -} - -///////////////// Asynchronous Callbacks //////////////////////// - -void FetchingStorage::PrefetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - char* buf = - const_cast(reinterpret_cast(req->aio_buf)); - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - State prev_state = latch->state; - latch->state = IN_MEMORY; - pthread_mutex_unlock(&latch->lock_); - /* Nothing interfered with our fetch */ - if (prev_state == FETCHING) { - string* value = new string(buf, PAGE_SIZE); - store->main_memory_->PutObject(key, value); - } - delete[] buf; - close(req->aio_fildes); - delete req; - } -} - -void FetchingStorage::UnfetchCompletionHandler(sigval_t sigval) { - struct aiocb *req; - req = (struct aiocb *)sigval.sival_ptr; - /* Did the request complete? */ - if (aio_error(req) == 0) { - /* Request completed successfully, get the return status */ - string key; - GetKey(req->aio_fildes, &key); - FetchingStorage* store = FetchingStorage::BuildStorage(); - Latch* latch = store->LatchFor(key); - pthread_mutex_lock(&latch->lock_); - // Hasn't been fetched. - assert(latch->active_requests >= 0); - int active_requests = latch->active_requests; - State state = latch->state; - pthread_mutex_unlock(&latch->lock_); - if (state == RELEASING && active_requests <= 0) { - store->main_memory_->DeleteObject(key); - latch->state = ON_DISK; - } - close(req->aio_fildes); - delete[] reinterpret_cast(req->aio_buf); - delete req; - } - return; -} - -/* - * Here live the bogus hacks. - */ - -bool FetchingStorage::FileRead(const Key& key, char* result, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - int fd = open(fileName.c_str(), O_RDONLY|O_NONBLOCK); - if (fd == -1) - return false; - return aio_read(generateControlBlock(fd, result, size, FETCH)) >= 0; -} - -bool FetchingStorage::FilePut(const Key& key, char* value, int size) { - string fileName(STORAGE_PATH); - fileName.append(key); - mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - int fd = open(fileName.c_str(), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, mode); - if (fd == -1) - return false; - return aio_write(generateControlBlock(fd, value, size, RELEASE)) >= 0; -} - -aiocb* FetchingStorage::generateControlBlock( - int fd, char* buf, const int size, Operation op) { - aiocb* aiocbp = new aiocb(); - aiocbp->aio_fildes = fd; - aiocbp->aio_offset = 0; - aiocbp->aio_buf = buf; - aiocbp->aio_nbytes = size; - aiocbp->aio_reqprio = 0; - /* Link the AIO request with a thread callback */ - aiocbp->aio_sigevent.sigev_notify = SIGEV_THREAD; - if (op == FETCH) - aiocbp->aio_sigevent.sigev_notify_function = - &PrefetchCompletionHandler; - else - aiocbp->aio_sigevent.sigev_notify_function = - &UnfetchCompletionHandler; - aiocbp->aio_sigevent.sigev_notify_attributes = NULL; - aiocbp->aio_sigevent.sigev_value.sival_ptr = aiocbp; - return aiocbp; -} diff --git a/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.h b/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.h deleted file mode 100755 index 273dc04c..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/fetching_storage.h +++ /dev/null @@ -1,93 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) -// -// An implementation of the storage interface taking into account -// main memory, disk, and swapping algorithms. - -#ifndef _DB_BACKEND_FETCHING_STORAGE_H_ -#define _DB_BACKEND_FETCHING_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/utils.h" -#include "backend/storage.h" -#include "backend/simple_storage.h" - -#define PAGE_SIZE 1000 -#define STORAGE_PATH "../db/storage/" - -#define COLD_CUTOFF 990000 - -class FetchingStorage : public Storage { - public: - static FetchingStorage* BuildStorage(); - ~FetchingStorage(); - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - virtual bool Prefetch(const Key &key, double* wait_time); - virtual bool Unfetch(const Key &key); - bool HardUnfetch(const Key& key); - - // Latch object that stores a counter for readlocks and a boolean for write - // locks. - enum State { - UNINITIALIZED, IN_MEMORY, ON_DISK, FETCHING, RELEASING - }; - class Latch { - public: - int active_requests; // Can be as many as you want. - State state; - pthread_mutex_t lock_; - - Latch() { - active_requests = 0; - state = UNINITIALIZED; - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const Key &key); - static void PrefetchCompletionHandler(sigval_t sigval); - static void UnfetchCompletionHandler(sigval_t sigval); - static void GetKey(int fd, Key* key); - - private: - FetchingStorage(); - /* - * The following functions are bogus hacks that should be factored out into - * a separate disk layer. To make the callbacks play nice for now, everything - * is being placed here. After November 1, we can swap out backends. - */ - - enum Operation { - FETCH, RELEASE - }; - - // Registers an asynchronous read. - bool FileRead(const Key& key, char* result, int size); - - // Registers an asynchronous write. - bool FilePut(const Key& key, char* value, int size); - - // XXX(scw): Document? `int fd' used to be `int& fd'. Move it to the last - // argument and retype `int* fd' if used as an output argument. - aiocb* generateControlBlock(int fd, char* buf, const int size, Operation op); - - static FetchingStorage* self; - - // Additional State - - Storage* main_memory_; - Latch* latches_; - - // GC thread stuff. - static void* RunGCThread(void *arg); - pthread_t gc_thread_; -}; -#endif // _DB_BACKEND_FETCHING_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.cc b/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.cc deleted file mode 100755 index 4d7760d4..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#include "backend/simple_storage.h" - -Value* SimpleStorage::ReadObject(const Key& key, int64 txn_id) { - if (objects_.count(key) != 0) { - return objects_[key]; - } else { - return NULL; - } -} - -bool SimpleStorage::PutObject(const Key& key, Value* value, int64 txn_id) { - pthread_mutex_lock(&mutex_); - objects_[key] = value; - pthread_mutex_unlock(&mutex_); - return true; -} - -bool SimpleStorage::DeleteObject(const Key& key, int64 txn_id) { - objects_.erase(key); - return true; -} - -void SimpleStorage::Initmutex() { - pthread_mutex_init(&mutex_, NULL); -} diff --git a/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.h b/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.h deleted file mode 100755 index f15f69e9..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/simple_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// A simple implementation of the storage interface using an stl map. - -#ifndef _DB_BACKEND_SIMPLE_STORAGE_H_ -#define _DB_BACKEND_SIMPLE_STORAGE_H_ - -#include - -#include "backend/storage.h" -#include "common/types.h" -#include - -using std::tr1::unordered_map; - -class SimpleStorage : public Storage { - public: - virtual ~SimpleStorage() {} - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } - virtual Value* ReadObject(const Key& key, int64 txn_id = 0); - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0); - virtual bool DeleteObject(const Key& key, int64 txn_id = 0); - - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex(); - - private: - unordered_map objects_; - pthread_mutex_t mutex_; - -}; -#endif // _DB_BACKEND_SIMPLE_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/backend/storage.h b/src_traditional_2pl_2pc_4_partitions/backend/storage.h deleted file mode 100755 index a793360a..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/storage.h +++ /dev/null @@ -1,48 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// The Storage class provides an interface for writing and accessing data -// objects stored by the system. - -#ifndef _DB_BACKEND_STORAGE_H_ -#define _DB_BACKEND_STORAGE_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class Storage { - public: - virtual ~Storage() {} - - // Loads object specified by 'key' into memory if currently stored - // on disk, asynchronously or otherwise. - virtual bool Prefetch(const Key &key, double* wait_time) = 0; - - // Unfetch object on memory, writing it off to disk, asynchronously or - // otherwise. - virtual bool Unfetch(const Key &key) = 0; - - // If the object specified by 'key' exists, copies the object into '*result' - // and returns true. If the object does not exist, false is returned. - virtual Value* ReadObject(const Key& key, int64 txn_id = 0) = 0; - - // Sets the object specified by 'key' equal to 'value'. Any previous version - // of the object is replaced. Returns true if the write succeeds, or false if - // it fails for any reason. - virtual bool PutObject(const Key& key, Value* value, int64 txn_id = 0) = 0; - - // Removes the object specified by 'key' if there is one. Returns true if the - // deletion succeeds (or if no object is found with the specified key), or - // false if it fails for any reason. - virtual bool DeleteObject(const Key& key, int64 txn_id = 0) = 0; - - // TODO(Thad): Something here - virtual void PrepareForCheckpoint(int64 stable) {} - virtual int Checkpoint() { return 0; } - virtual void Initmutex() {} -}; - -#endif // _DB_BACKEND_STORAGE_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.cc b/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.cc deleted file mode 100755 index 79da935e..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.cc +++ /dev/null @@ -1,217 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -StorageManager::StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn, int partition_id) - : configuration_(config), connection_(connection), - actual_storage_(actual_storage), txn_(txn), partition_id_(partition_id) { - MessageProto message; - - // If reads are performed at this node, execute local reads and broadcast - // results to all (other) writers. -received_prepared_reply_number = 0; -received_prepared_message = false; - -received_commit_reply_number = 0; -received_commit_message = false; - - bool reader = false; - for (int i = 0; i < txn->readers_size(); i++) { - if (txn->readers(i) == partition_id_) - reader = true; - } - - if (reader) { - // message.set_destination_channel(IntToString(txn->txn_id())); - message.set_type(MessageProto::READ_RESULT); - - // Execute local reads. - for (int i = 0; i < txn->read_set_size(); i++) { - const Key& key = txn->read_set(i); - if (configuration_->LookupPartition(key) == - partition_id_) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - for (int i = 0; i < txn->read_write_set_size(); i++) { - const Key& key = txn->read_write_set(i); - if (configuration_->LookupPartition(key) == partition_id_) { - Value* val = actual_storage_->ReadObject(key); - objects_[key] = val; - message.add_keys(key); - message.add_values(val == NULL ? "" : *val); - } - } - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) != partition_id_) { - message.set_destination_node(txn->writers(i)); - string channel = IntToString(txn->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } - } - - // Note whether this node is a writer. If not, no need to do anything further. - writer = false; - for (int i = 0; i < txn->writers_size(); i++) { - if (txn->writers(i) == partition_id_) - writer = true; - } - - // Scheduler is responsible for calling HandleReadResponse. We're done here. -} - -void StorageManager::HandleReadResult(const MessageProto& message) { - assert(message.type() == MessageProto::READ_RESULT); - for (int i = 0; i < message.keys_size(); i++) { - Value* val = new Value(message.values(i)); - objects_[message.keys(i)] = val; - remote_reads_.push_back(val); - } -} - -bool StorageManager::ReadyToExecute() { - return static_cast(objects_.size()) == - txn_->read_set_size() + txn_->read_write_set_size(); -} -// -----------FOR 2PC prepard phase------------------ -bool StorageManager::ReceivedAllPreparedReplyMessages() { - return received_prepared_reply_number == txn_->writers_size() - 1; -} - -void StorageManager::BroadcastPreparedMessages() { - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn_->writers_size(); i++) { - if (txn_->writers(i) != partition_id_) { - MessageProto message; - message.set_type(MessageProto::PREPARED); - message.set_destination_node(txn_->writers(i)); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } -} - -void StorageManager::HandlePreparedReplyMessages(const MessageProto& message) { - assert(message.type() == MessageProto::PREPARED_REPLY); - received_prepared_reply_number ++; -} - -bool StorageManager::ReceivedPreparedMessages() { - return received_prepared_message == true; -} - -void StorageManager::HandlePreparedMessages(const MessageProto& message) { - assert(message.type() == MessageProto::PREPARED); - received_prepared_message = true; -} - -void StorageManager::SendPreparedReplyMessages() { - MessageProto message; - message.set_type(MessageProto::PREPARED_REPLY); - // Broadcast local reads to (other) writers. - - message.set_destination_node(txn_->txn_node()); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); -} -// -----------FOR 2PC prepard phase------------------ - - -// -----------FOR 2PC commit phase------------------ -bool StorageManager::ReceivedAllCommitReplyMessages() { - return received_commit_reply_number == txn_->writers_size() - 1; -} - -void StorageManager::BroadcastCommitMessages() { - - // Broadcast local reads to (other) writers. - for (int i = 0; i < txn_->writers_size(); i++) { - if (txn_->writers(i) != partition_id_) { - MessageProto message; - message.set_type(MessageProto::COMMIT); - message.set_destination_node(txn_->writers(i)); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); - } - } -} - -void StorageManager::HandleCommitReplyMessages(const MessageProto& message) { - assert(message.type() == MessageProto::COMMIT_REPLY); - received_commit_reply_number ++; -} - -bool StorageManager::ReceivedCommitMessages() { - return received_commit_message == true; -} - -void StorageManager::HandleCommitMessages(const MessageProto& message) { - assert(message.type() == MessageProto::COMMIT); - received_commit_message = true; -} - -void StorageManager::SendCommitReplyMessages() { - MessageProto message; - message.set_type(MessageProto::COMMIT_REPLY); - // Broadcast local reads to (other) writers. - - message.set_destination_node(txn_->txn_node()); - string channel = IntToString(txn_->txn_id()); - message.set_destination_channel(channel); - connection_->Send1(message); -} -// -----------FOR 2PC commit phase------------------ - - - - -StorageManager::~StorageManager() { - for (vector::iterator it = remote_reads_.begin(); - it != remote_reads_.end(); ++it) { - delete *it; - } -} - -Value* StorageManager::ReadObject(const Key& key) { - return objects_[key]; -} - -bool StorageManager::PutObject(const Key& key, Value* value) { - // Write object to storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->PutObject(key, value, txn_->txn_id()); - else - return true; // Not this node's problem. -} - -bool StorageManager::DeleteObject(const Key& key) { - // Delete object from storage if applicable. - if (configuration_->LookupPartition(key) == partition_id_) - return actual_storage_->DeleteObject(key, txn_->txn_id()); - else - return true; // Not this node's problem. -} - diff --git a/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.h b/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.h deleted file mode 100755 index 9e748498..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/storage_manager.h +++ /dev/null @@ -1,112 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A wrapper for a storage layer that can be used by an Application to simplify -// application code by hiding all inter-node communication logic. By using this -// class as the primary interface for applications to interact with storage of -// actual data objects, applications can be written without paying any attention -// to partitioning at all. -// -// StorageManager use: -// - Each transaction execution creates a new StorageManager and deletes it -// upon completion. -// - No ReadObject call takes as an argument any value that depends on the -// result of a previous ReadObject call. -// - In any transaction execution, a call to DoneReading must follow ALL calls -// to ReadObject and must precede BOTH (a) any actual interaction with the -// values 'read' by earlier calls to ReadObject and (b) any calls to -// PutObject or DeleteObject. - -#ifndef _DB_BACKEND_STORAGE_MANAGER_H_ -#define _DB_BACKEND_STORAGE_MANAGER_H_ - -#include - -#include -#include - -#include "common/types.h" -#include "backend/storage.h" - -using std::vector; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class MessageProto; -class Scheduler; -class Storage; -class TxnProto; - -class StorageManager { - public: - // TODO(alex): Document this class correctly. - StorageManager(Configuration* config, Connection* connection, - Storage* actual_storage, TxnProto* txn, int partition_id); - - ~StorageManager(); - - Value* ReadObject(const Key& key); - bool PutObject(const Key& key, Value* value); - bool DeleteObject(const Key& key); - - void HandleReadResult(const MessageProto& message); - bool ReadyToExecute(); - - Storage* GetStorage() { return actual_storage_; } - - bool ReceivedAllPreparedReplyMessages(); - void BroadcastPreparedMessages(); - void HandlePreparedReplyMessages(const MessageProto& message); - bool ReceivedPreparedMessages(); - void HandlePreparedMessages(const MessageProto& message); - void SendPreparedReplyMessages(); - - bool ReceivedAllCommitReplyMessages(); - void BroadcastCommitMessages(); - void HandleCommitReplyMessages(const MessageProto& message); - bool ReceivedCommitMessages(); - void HandleCommitMessages(const MessageProto& message); - void SendCommitReplyMessages(); - - // Set by the constructor, indicating whether 'txn' involves any writes at - // this node. - bool writer; - -// private: - friend class DeterministicScheduler; - - // Pointer to the configuration object for this node. - Configuration* configuration_; - - // A Connection object that can be used to send and receive messages. - Connection* connection_; - - // Storage layer that *actually* stores data objects on this node. - Storage* actual_storage_; - - // Transaction that corresponds to this instance of a StorageManager. - TxnProto* txn_; - - int partition_id_; - - // Local copy of all data objects read/written by 'txn_', populated at - // StorageManager construction time. - // - // TODO(alex): Should these be pointers to reduce object copying overhead? - unordered_map objects_; - - vector remote_reads_; - - int received_prepared_reply_number; - - bool received_prepared_message; - - int received_commit_reply_number; - - bool received_commit_message; - - -}; - -#endif // _DB_BACKEND_STORAGE_MANAGER_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/backend/versioned_storage.h b/src_traditional_2pl_2pc_4_partitions/backend/versioned_storage.h deleted file mode 100755 index 24c6223b..00000000 --- a/src_traditional_2pl_2pc_4_partitions/backend/versioned_storage.h +++ /dev/null @@ -1,46 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// -// This is an abstract interface to a versioned storage system -// (credit: comments are exactly the same as they are for storage.h) - -#ifndef _DB_BACKEND_VERSIONED_STORAGE_H_ -#define _DB_BACKEND_VERSIONED_STORAGE_H_ - -#include - -#include "backend/storage.h" - -using std::vector; - -class VersionedStorage : public Storage { - public: - virtual ~VersionedStorage() {} - - // Read object takes a transaction id and places the value at time t <= txn_id - // into the address pointed to by result - virtual Value* ReadObject(const Key& key, int64 txn_id) = 0; - - // Put object takes a value and adds a version to the linked list storage - // with the specified txn_id as a timestamp - virtual bool PutObject(const Key& key, Value* value, int64 txn_id) = 0; - - // The delete method actually merely places an empty string at the version - // specified by txn_id. This is in contrast to actually deleting a value - // (which versioned storage never does. - virtual bool DeleteObject(const Key& key, int64 txn_id) = 0; - - // TODO(Thad): We should really make this a virtually required interface for - // all storage classes but to avoid conflicts I'm just gonna leave it in the - // versioned storage class. - // This method is a requirement for versioned storages to implement. It - // causes a snapshot to be performed at the virtually consistent state - // specified (i.e. txn_id). - virtual int Checkpoint() = 0; - - - // TODO(Thad): Implement something real here - virtual bool Prefetch(const Key &key, double* wait_time) { return false; } - virtual bool Unfetch(const Key &key) { return false; } -}; - -#endif // _DB_BACKEND_VERSIONED_STORAGE_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/common/.DS_Store b/src_traditional_2pl_2pc_4_partitions/common/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/common/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/common/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/common/Makefile.inc deleted file mode 100755 index 6f46fd07..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += common - -# Create template specific variables -UPPERC_DIR := COMMON -LOWERC_DIR := common - -COMMON_SRCS := common/configuration.cc \ - common/connection.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-common: $(COMMON_TESTS) - @for a in $(COMMON_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/common/configuration.h b/src_traditional_2pl_2pc_4_partitions/common/configuration.h deleted file mode 100755 index 6d1d256f..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/configuration.h +++ /dev/null @@ -1,101 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Each node in the system has a Configuration, which stores the identity of -// that node, the system's current execution mode, and the set of all currently -// active nodes in the system. -// -// Config file format: -// # (Lines starting with '#' are comments.) -// # List all nodes in the system. -// # Node=:::: -// node13=1:3:16:4.8.15.16:1001:1002 -// node23=2:3:16:4.8.15.16:1004:1005 -// -// Note: Epoch duration, application and other global global options are -// specified as command line options at invocation time (see -// deployment/main.cc). - -#ifndef _DB_COMMON_CONFIGURATION_H_ -#define _DB_COMMON_CONFIGURATION_H_ - -#include - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; -using std::vector; -using std::tr1::unordered_map; -// The work thread number of one node -#define WorkersNumber 5 -#define SPINTIME 0.000001 - -extern map latest_order_id_for_customer; -extern map latest_order_id_for_district; -extern map smallest_order_id_for_district; -extern map customer_for_order; -extern unordered_map next_order_id_for_district; -extern map item_for_order_line; -extern map order_line_number; - -extern vector* involed_customers; - -extern pthread_mutex_t mutex_; -extern pthread_mutex_t mutex_for_item; - -#define ORDER_LINE_NUMBER 10 - - -struct Node { - - int node_id; - int replica_id; - // Globally unique partition identifier. - int partition_id; - // IP address of this node's machine. - string host; - - // Port on which to listen for messages from other nodes. - int port; - - // Total number of cores available for use by this node. - // Note: Is this needed? - int cores; -}; - -class Configuration { - public: - Configuration(int node_id, const string& filename); - - // Returns the node_id of the partition at which 'key' is stored. - int LookupPartition(const Key& key) const; - - // Dump the current config into the file in key=value format. - // Returns true when success. - bool WriteToFile(const string& filename) const; - - // This node's node_id. - int this_node_id; - - // Tracks the set of current active nodes in the system. - map all_nodes; - - // Tracks the set of current active partitions in the system. -// map all_partitions; - // partition number of this node -// vector partitions_of_this_node; - - private: - // TODO(alex): Comments. - void ProcessConfigLine(char key[], char value[]); - int ReadFromFile(const string& filename); -}; - -#endif // _DB_COMMON_CONFIGURATION_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/common/configuration_test.conf b/src_traditional_2pl_2pc_4_partitions/common/configuration_test.conf deleted file mode 100755 index f3af20f5..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/configuration_test.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Node=:::: -node1=0:1:16:128.36.232.50:50001 -node2=0:2:16:128.36.232.50:50002 - diff --git a/src_traditional_2pl_2pc_4_partitions/common/configuration_test_one_node.conf b/src_traditional_2pl_2pc_4_partitions/common/configuration_test_one_node.conf deleted file mode 100755 index 6f5083ba..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/configuration_test_one_node.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Node=:::: -node0=0:0:16:128.36.232.50:60001 diff --git a/src_traditional_2pl_2pc_4_partitions/common/connection.cc b/src_traditional_2pl_2pc_4_partitions/common/connection.cc deleted file mode 100755 index 06515275..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/connection.cc +++ /dev/null @@ -1,401 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/connection.h" - -#include -#include - -#include "common/configuration.h" -#include "common/utils.h" -#include "sequencer/sequencer.h" - -using zmq::socket_t; - -ConnectionMultiplexer::ConnectionMultiplexer(Configuration* config) - : configuration_(config), context_(1), new_connection_channel_(NULL), - delete_connection_channel_(NULL), deconstructor_invoked_(false) { - // Lookup port. (Pick semi-arbitrary port if node id < 0). - if (config->this_node_id < 0) - port_ = config->all_nodes.begin()->second->port; - else - port_ = config->all_nodes.find(config->this_node_id)->second->port; - - // Bind local (inproc) incoming socket. - inproc_in_ = new socket_t(context_, ZMQ_PULL); - inproc_in_->bind("inproc://__inproc_in_endpoint__"); - - // Bind port for remote incoming socket. - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "tcp://*:%d", port_); - remote_in_ = new socket_t(context_, ZMQ_PULL); - remote_in_->bind(endpoint); - - // Wait for other nodes to bind sockets before connecting to them. - Spin(0.1); - - send_mutex_ = new pthread_mutex_t[(int)config->all_nodes.size()]; - - // Connect to remote outgoing sockets. - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - if (it->second->node_id != config->this_node_id) { // Only remote nodes. - snprintf(endpoint, sizeof(endpoint), "tcp://%s:%d", - it->second->host.c_str(), it->second->port); - remote_out_[it->second->node_id] = new socket_t(context_, ZMQ_PUSH); - remote_out_[it->second->node_id]->connect(endpoint); - pthread_mutex_init(&send_mutex_[it->second->node_id], NULL); - } - } - -cpu_set_t cpuset; -pthread_attr_t attr; -pthread_attr_init(&attr); -//pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - -CPU_ZERO(&cpuset); -CPU_SET(3, &cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - - - // Start Multiplexer main loop running in background thread. - pthread_create(&thread_, &attr, RunMultiplexer, reinterpret_cast(this)); - - // Initialize mutex for future calls to NewConnection. - pthread_mutex_init(&new_connection_mutex_, NULL); - new_connection_channel_ = NULL; - - // Just to be safe, wait a bit longer for all other nodes to finish - // multiplexer initialization before returning to the caller, who may start - // sending messages immediately. - Spin(0.1); -} - -ConnectionMultiplexer::~ConnectionMultiplexer() { - // Stop the multixplexer's main loop. - deconstructor_invoked_ = true; - pthread_join(thread_, NULL); - - // Close tcp sockets. - delete remote_in_; - for (unordered_map::iterator it = remote_out_.begin(); - it != remote_out_.end(); ++it) { - delete it->second; - } - - // Close inproc sockets. - delete inproc_in_; - for (unordered_map::iterator it = inproc_out_.begin(); - it != inproc_out_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = remote_result_.begin(); - it != remote_result_.end(); ++it) { - delete it->second; - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - delete it->second; - } - -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - // Register the new connection request. - new_connection_channel_ = &channel; - - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - - return connection; -} - -Connection* ConnectionMultiplexer::NewConnection(const string& channel, AtomicQueue** aa) { - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&new_connection_mutex_); - - remote_result_[channel] = *aa; - // Register the new connection request. - new_connection_channel_ = &channel; - // Wait for the Run() loop to create the Connection object. (It will reset - // new_connection_channel_ to NULL when the new connection has been created. - while (new_connection_channel_ != NULL) {} - Connection* connection = new_connection_; - new_connection_ = NULL; - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&new_connection_mutex_); - return connection; -} - -void ConnectionMultiplexer::Run() { - MessageProto message; - zmq::message_t msg; - - while (!deconstructor_invoked_) { - // Serve any pending NewConnection request. - if (new_connection_channel_ != NULL) { - if (inproc_out_.count(*new_connection_channel_) > 0) { - // Channel name already in use. Report an error and set new_connection_ - // (which NewConnection() will return) to NULL. - std::cerr << "Attempt to create channel that already exists: " - << (*new_connection_channel_) << "\n" << std::flush; - new_connection_ = NULL; - } else { - // Channel name is not already in use. Create a new Connection object - // and connect it to this multiplexer. - new_connection_ = new Connection(); - new_connection_->channel_ = *new_connection_channel_; - new_connection_->multiplexer_ = this; - char endpoint[256]; - snprintf(endpoint, sizeof(endpoint), "inproc://%s", - new_connection_channel_->c_str()); - inproc_out_[*new_connection_channel_] = - new socket_t(context_, ZMQ_PUSH); - inproc_out_[*new_connection_channel_]->bind(endpoint); - new_connection_->socket_in_ = new socket_t(context_, ZMQ_PULL); - new_connection_->socket_in_->connect(endpoint); - new_connection_->socket_out_ = new socket_t(context_, ZMQ_PUSH); - new_connection_->socket_out_ - ->connect("inproc://__inproc_in_endpoint__"); - - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[*new_connection_channel_].begin(); - i != undelivered_messages_[*new_connection_channel_].end(); ++i) { - Send(*i); - } - undelivered_messages_.erase(*new_connection_channel_); - } - - if (new_connection_channel_->substr(0, 6) == "worker") { - link_unlink_queue_[*new_connection_channel_] = new AtomicQueue(); - } - // Reset request variable. - new_connection_channel_ = NULL; - } - - // Serve any pending (valid) connection deletion request. - if (delete_connection_channel_ != NULL && - inproc_out_.count(*delete_connection_channel_) > 0) { - delete inproc_out_[*delete_connection_channel_]; - inproc_out_.erase(*delete_connection_channel_); - delete_connection_channel_ = NULL; - // TODO(alex): Should we also be emptying deleted channels of messages - // and storing them in 'undelivered_messages_' in case the channel is - // reopened/relinked? Probably. - } - - // Forward next message from a remote node (if any). - if (remote_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - Send(message); - } - - // Forward next message from a local component (if any), intercepting - // local Link/UnlinkChannel requests. - if (inproc_in_->recv(&msg, ZMQ_NOBLOCK)) { - message.ParseFromArray(msg.data(), msg.size()); - // Normal message. Forward appropriately. - Send(message); - } - - for (unordered_map*>::iterator it = link_unlink_queue_.begin(); - it != link_unlink_queue_.end(); ++it) { - MessageProto message; - bool got_it = it->second->Pop(&message); - if (got_it == true) { - if (message.type() == MessageProto::LINK_CHANNEL) { - remote_result_[message.channel_request()] = remote_result_[it->first]; - // Forward on any messages sent to this channel before it existed. - vector::iterator i; - for (i = undelivered_messages_[message.channel_request()].begin(); - i != undelivered_messages_[message.channel_request()].end(); - ++i) { - Send(*i); - } - undelivered_messages_.erase(message.channel_request()); - if (doing_deadlocks.count(message.channel_request()) > 0) { - MessageProto message1; - message1.set_type(MessageProto::TXN_ABORT); - message1.set_destination_channel(message.channel_request()); - message1.set_destination_node(configuration_->this_node_id); - Send(message1); - } - } else if (message.type() == MessageProto::UNLINK_CHANNEL) { - remote_result_.erase(message.channel_request()); - } - } - } - - } -} - -// Function to call multiplexer->Run() in a new pthread. -void* ConnectionMultiplexer::RunMultiplexer(void *multiplexer) { - reinterpret_cast(multiplexer)->Run(); - return NULL; -} - -void ConnectionMultiplexer::Send(const MessageProto& message) { - if ((message.destination_node() == configuration_->this_node_id) && (message.type() == MessageProto::TXN_ABORT)) { - if (undelivered_messages_.count(message.destination_channel()) > 0) { - undelivered_messages_.erase(message.destination_channel()); - remote_result_["force_abort_unactive_txn"]->Push(message); - doing_deadlocks[message.destination_channel()] = 1; - } else { - if (remote_result_.count(message.destination_channel()) == 0) { - remote_result_["force_abort_unactive_txn"]->Push(message); - doing_deadlocks[message.destination_channel()] = 1; - } else { - remote_result_[message.destination_channel()]->Push(message); - } - } - return ; - } - - if ((message.destination_node() == configuration_->this_node_id) && (message.type() == MessageProto::READ_RESULT || message.type() == MessageProto::PREPARED || - message.type() == MessageProto::PREPARED_REPLY || message.type() == MessageProto::COMMIT || - message.type() == MessageProto::COMMIT_REPLY || message.type() == MessageProto::WAIT_FOR_GRAPH)) { - if (remote_result_.count(message.destination_channel()) > 0) { - remote_result_[message.destination_channel()]->Push(message); - } else { - undelivered_messages_[message.destination_channel()].push_back(message); - } - } else { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - - // Send message. - if (message.destination_node() == configuration_->this_node_id) { - // Message is addressed to a local channel. If channel is valid, send the - // message on, else store it to be delivered if the channel is ever created. - if (inproc_out_.count(message.destination_channel()) > 0) - inproc_out_[message.destination_channel()]->send(msg); - else - undelivered_messages_[message.destination_channel()].push_back(message); - - } else { - // Message is addressed to valid remote node. Channel validity will be - // checked by the remote multiplexer. - pthread_mutex_lock(&send_mutex_[message.destination_node()]); - remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&send_mutex_[message.destination_node()]); - } - } -} - -Connection::~Connection() { - // Unlink any linked channels. - for (set::iterator it = linked_channels_.begin(); - it != linked_channels_.end(); ++it) { - UnlinkChannel(*it); - } - - // Disallow concurrent calls to NewConnection/~Connection. - pthread_mutex_lock(&(multiplexer_->new_connection_mutex_)); - - // Delete socket on Connection end. - delete socket_in_; - delete socket_out_; - - // Prompt multiplexer to delete socket on its end. - multiplexer_->delete_connection_channel_ = &channel_; - - // Wait for the Run() loop to delete its socket for this Connection object. - // (It will then reset delete_connection_channel_ to NULL.) - while (multiplexer_->delete_connection_channel_ != NULL) {} - - // Allow future calls to NewConnection/~Connection. - pthread_mutex_unlock(&(multiplexer_->new_connection_mutex_)); -} - -void Connection::Send(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - // Send message. - socket_out_->send(msg); -} - -void Connection::Send1(const MessageProto& message) { - // Prepare message. - string* message_string = new string(); - message.SerializeToString(message_string); - zmq::message_t msg(reinterpret_cast( - const_cast(message_string->data())), - message_string->size(), - DeleteString, - message_string); - pthread_mutex_lock(&multiplexer()->send_mutex_[message.destination_node()]); - multiplexer()->remote_out_[message.destination_node()]->send(msg); - pthread_mutex_unlock(&multiplexer()->send_mutex_[message.destination_node()]); -} - -bool Connection::GetMessage(MessageProto* message) { - zmq::message_t msg_; - if (socket_in_->recv(&msg_, ZMQ_NOBLOCK)) { - // Received a message. - message->ParseFromArray(msg_.data(), msg_.size()); - return true; - } else { - // No message received at this time. - return false; - } -} - -bool Connection::GetMessageBlocking(MessageProto* message, - double max_wait_time) { - double start = GetTime(); - do { - if (GetMessage(message)) { - // Received a message. - return true; - } - } while (GetTime() < start + max_wait_time); - - // Waited for max_wait_time, but no message was received. - return false; -} - -void Connection::LinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::LINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - -void Connection::UnlinkChannel(const string& channel) { - MessageProto m; - m.set_type(MessageProto::UNLINK_CHANNEL); - m.set_channel_request(channel); - multiplexer()->link_unlink_queue_[channel_]->Push(m); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/common/connection.h b/src_traditional_2pl_2pc_4_partitions/common/connection.h deleted file mode 100755 index f8d52166..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/connection.h +++ /dev/null @@ -1,198 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Library for handling messaging between system nodes. Each node generally owns -// a ConnectionMultiplexer object as well as a Configuration object. - -#ifndef _DB_COMMON_CONNECTION_H_ -#define _DB_COMMON_CONNECTION_H_ - -#include - -#include -#include -#include -#include -#include - - -#include "common/zmq.hpp" -#include "proto/message.pb.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::map; -using std::set; -using std::string; -using std::vector; -using std::tr1::unordered_map; - -class Configuration; - -// TODO(alex): What if a multiplexer receives a message sent to a local channel -// that doesn't exist (yet)? -class Connection; -class ConnectionMultiplexer { - public: - // Create a ConnectionMultiplexer that establishes two-way communication with - // Connections for every other node specified by '*config' to exist. - explicit ConnectionMultiplexer(Configuration* config); - - // TODO(alex): The deconstructor currently closes all sockets. Connection - // objects, however, do not have a defined behavior for trying to - // send messages to the multiplexor after it has been destroyed. - ~ConnectionMultiplexer(); - - // Creates and registers a new connection with channel name 'channel', unless - // the channel name is already in use, in which case NULL is returned. The - // caller (not the multiplexer) owns of the newly created Connection object. - Connection* NewConnection(const string& channel); - - Connection* NewConnection(const string& channel, AtomicQueue** aa); - - zmq::context_t* context() { return &context_; } - - private: - friend class Connection; - - // Runs the Multiplexer's main loop. Run() is called in a new thread by the - // constructor. - void Run(); - - // Function to call multiplexer->Run() in a new pthread. - static void* RunMultiplexer(void *multiplexer); - - // TODO(alex): Comments. - void Send(const MessageProto& message); - - // Separate pthread context in which to run the multiplexer's main loop. - pthread_t thread_; - - // Pointer to Configuration instance used to construct this Multiplexer. - // (Currently used primarily for finding 'this_node_id'.) - Configuration* configuration_; - - // Context shared by all Connection objects with channels to this - // multiplexer. - zmq::context_t context_; - - // Port on which to listen for incoming messages from other nodes. - int port_; - - // Socket listening for messages from other nodes. Type = ZMQ_PULL. - zmq::socket_t* remote_in_; - - // Sockets for outgoing traffic to other nodes. Keyed by node_id. - // Type = ZMQ_PUSH. - unordered_map remote_out_; - - // Socket listening for messages from Connections. Type = ZMQ_PULL. - zmq::socket_t* inproc_in_; - - // Sockets for forwarding messages to Connections. Keyed by channel - // name. Type = ZMQ_PUSH. - unordered_map inproc_out_; - - unordered_map*> remote_result_; - - unordered_map*> link_unlink_queue_; - - // Stores messages addressed to local channels that do not exist at the time - // the message is received (so that they may be delivered if a connection is - // ever created with the specified channel name). - // - // TODO(alex): Prune this occasionally? - unordered_map > undelivered_messages_; - - // Protects concurrent calls to NewConnection(). - pthread_mutex_t new_connection_mutex_; - - pthread_mutex_t* send_mutex_; - - // Specifies a requested channel. Null if there is no outstanding new - // connection request. - const string* new_connection_channel_; - - // Specifies channel requested to be deleted. Null if there is no outstanding - // connection deletion request. - const string* delete_connection_channel_; - - // Pointer to Connection objects recently created in the Run() thread. - Connection* new_connection_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // DISALLOW_COPY_AND_ASSIGN - ConnectionMultiplexer(const ConnectionMultiplexer&); - ConnectionMultiplexer& operator=(const ConnectionMultiplexer&); -}; - -class Connection { - public: - // Closes all sockets. - ~Connection(); - - // Sends 'message' to the Connection specified by - // 'message.destination_node()' and 'message.destination_channel()'. - void Send(const MessageProto& message); - - void Send1(const MessageProto& message); - - // Loads the next incoming MessageProto into 'message'. Returns true, unless - // no message is queued up to be delivered, in which case false is returned. - // 'message->Clear()' is NOT called. Non-blocking. - bool GetMessage(MessageProto* message); - - // Loads the next incoming MessageProto into 'message'. If no message is - // queued up to be delivered, GetMessageBlocking waits at most 'max_wait_time' - // seconds for a message to arrive. If no message arrives, false is returned. - // 'message->Clear()' is NOT called. - bool GetMessageBlocking(MessageProto* message, double max_wait_time); - - // Links 'channel' to this Connection object so that messages sent to - // 'channel' will be forwarded to this Connection. - // - // Requires: The requested channel name is not already in use. - void LinkChannel(const string& channel); - - // Links 'channel' from this Connection object so that messages sent to - // 'channel' will no longer be forwarded to this Connection. - // - // Requires: The requested channel name was previously linked to this - // Connection by LinkChannel. - void UnlinkChannel(const string& channel); - - // Returns a pointer to this Connection's multiplexer. - ConnectionMultiplexer* multiplexer() { return multiplexer_; } - - // Return a const ref to this Connection's channel name. - const string& channel() { return channel_; } - - private: - friend class ConnectionMultiplexer; - - // Channel name that 'multiplexer_' uses to identify which messages to - // forward to this Connection object. - string channel_; - - // Additional channels currently linked to this Connection object. - set linked_channels_; - - // Pointer to the main ConnectionMultiplexer with which the Connection - // communicates. Not owned by the Connection. - ConnectionMultiplexer* multiplexer_; - - // Socket for sending messages to 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_out_; - - // Socket for getting messages from 'multiplexer_'. Type = ZMQ_PUSH. - zmq::socket_t* socket_in_; - - zmq::message_t msg_; -}; - -#endif // _DB_COMMON_CONNECTION_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/common/test_template.cc b/src_traditional_2pl_2pc_4_partitions/common/test_template.cc deleted file mode 100755 index 9dc11631..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/test_template.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Author: Name (email) - -#include "common/testing.h" - -TEST(MyTest) { - // Testing code here ... - - END; -} - -int main(int argc, char** argv) { - MyTest(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/common/testing.h b/src_traditional_2pl_2pc_4_partitions/common/testing.h deleted file mode 100755 index 83e27c03..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/testing.h +++ /dev/null @@ -1,96 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// NOTE(scw): This file is deprecated. The project is migrating to googletest. -// -// Testing framework similar to Google unittests. Example: -// -// TEST(SampleObjectTest) { -// SampleObject a(4,8,15); -// SampleObject b(16,23,42); -// -// EXPECT_FALSE(a.TransformCalled()); -// -// a.Transform(); -// EXPECT_TRUE(a.TransformCalled()); -// -// EXPECT_EQ(b,a); -// -// END; -// } -// -// int main(int argc, char** argv) { -// SampleObjectTest(); -// } -// - -#ifndef _DB_COMMON_TESTING_H_ -#define _DB_COMMON_TESTING_H_ - -#warning Using deprecated common/test.h module, use googletest instead. - -#include -#include - -using namespace std; // Don't do this at home, kids. - -// Global variable tracking whether current test has failed. -bool __failed_; - -#define WARN(MSG) printf("%s:%d: %s\n", __FILE__, __LINE__, MSG) -#define CHECK(T,MSG) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - WARN(MSG); \ - } \ - } while (0) - -#define LINE \ - cout << "[ " << __FUNCTION__ << " ] " - -#define EXPECT_TRUE(T) \ - do { \ - if (!(T)) { \ - __failed_ = true; \ - cout << "EXPECT_TRUE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_FALSE(T) \ - do { \ - if (T) { \ - __failed_ = true; \ - cout << "EXPECT_FALSE(" << #T << ") failed at " \ - << __FILE__ << ":" << __LINE__ << "\n"; \ - } \ - } while (0) - -#define EXPECT_EQ(A,B) \ - do { \ - if ((A) != (B)) { \ - __failed_ = true; \ - cout << "EXPECT_EQ(" << #A << ", " << #B \ - << ") \033[1;31mfailed\033[0m at " \ - << __FILE__ << ":" << __LINE__ << "\n" \ - << "Expected:\n" << A << "\n" \ - << "Actual:\n" << B << "\n"; \ - } \ - } while (0) - -#define TEST(TESTNAME) \ -void TESTNAME() { \ - __failed_ = false; \ - LINE << "\033[1;32mBEGIN\033[0m\n"; \ - do - -#define END \ - if (__failed_) { \ - LINE << "\033[1;31mFAIL\033[0m\n"; \ - } else { \ - LINE << "\033[1;32mPASS\033[0m\n"; \ - } \ - } while (0); - - -#endif // _DB_COMMON_TESTING_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/common/types.h b/src_traditional_2pl_2pc_4_partitions/common/types.h deleted file mode 100755 index c446e0c8..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/types.h +++ /dev/null @@ -1,60 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Defininitions of some common types and constants used in the system. - -#ifndef _DB_COMMON_TYPES_H_ -#define _DB_COMMON_TYPES_H_ - -#include - -#include - -using std::string; - -// Abbreviated signed int types. -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// Abbreviated unsigned int types. -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -// 'bytes' is an arbitrary sequence of bytes, represented as a string. -typedef string bytes; - -// Convenience functions for converting between (signed) int and 'bytes' types. -static inline bytes PackInt8 (int8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackInt16(int16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackInt32(int32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackInt64(int64 x) { return bytes((const char*)&x, 8); } -static inline int8 UnpackInt8 (bytes s) { return *((int8 *)(s.data())); } -static inline int16 UnpackInt16(bytes s) { return *((int16*)(s.data())); } -static inline int32 UnpackInt32(bytes s) { return *((int32*)(s.data())); } -static inline int64 UnpackInt64(bytes s) { return *((int64*)(s.data())); } - -// Convenience functions for converting between unsigned int and 'bytes' types. -static inline bytes PackUInt8 (uint8 x) { return bytes((const char*)&x, 1); } -static inline bytes PackUInt16(uint16 x) { return bytes((const char*)&x, 2); } -static inline bytes PackUInt32(uint32 x) { return bytes((const char*)&x, 4); } -static inline bytes PackUInt64(uint64 x) { return bytes((const char*)&x, 8); } -static inline uint8 UnpackUInt8 (bytes s) { return *((uint8 *)(s.data())); } -static inline uint16 UnpackUInt16(bytes s) { return *((uint16*)(s.data())); } -static inline uint32 UnpackUInt32(bytes s) { return *((uint32*)(s.data())); } -static inline uint64 UnpackUInt64(bytes s) { return *((uint64*)(s.data())); } - -// Key type for database objects. -// Note: if this changes from bytes, the types need to be updated for the -// following fields in .proto files: -// proto/txn.proto: -// TxnProto::'read_set' -// TxnProto::'write_set' -typedef bytes Key; - -// Value type for database objects. -typedef bytes Value; - -#endif // _DB_COMMON_TYPES_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/common/utils.h b/src_traditional_2pl_2pc_4_partitions/common/utils.h deleted file mode 100755 index fbdf09a0..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/utils.h +++ /dev/null @@ -1,381 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): UNIT TESTING! - -#ifndef _DB_COMMON_UTILS_H_ -#define _DB_COMMON_UTILS_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/types.h" - -using std::string; -using std::vector; -using std::tr1::unordered_map; - -#define ASSERTS_ON true - -#define DCHECK(ARG) do { if (ASSERTS_ON) assert(ARG); } while (0) - -// Status code for return values. -struct Status { - // Represents overall status state. - enum Code { - ERROR = 0, - OKAY = 1, - DONE = 2, - }; - Code code; - - // Optional explanation. - string message; - - // Constructors. - explicit Status(Code c) : code(c) {} - Status(Code c, const string& s) : code(c), message(s) {} - static Status Error() { return Status(ERROR); } - static Status Error(const string& s) { return Status(ERROR, s); } - static Status Okay() { return Status(OKAY); } - static Status Done() { return Status(DONE); } - - // Pretty printing. - string ToString() { - string out; - if (code == ERROR) out.append("Error"); - if (code == OKAY) out.append("Okay"); - if (code == DONE) out.append("Done"); - if (message.size()) { - out.append(": "); - out.append(message); - } - return out; - } -}; - -// Returns the number of seconds since midnight according to local system time, -// to the nearest microsecond. -static inline double GetTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec/1e6; -} - -// Busy-wait for 'duration' seconds. -static inline void Spin(double duration) { - usleep(1000000 * duration); -// double start = GetTime(); -// while (GetTime() < start + duration) {} -} - -// Busy-wait until GetTime() >= time. -static inline void SpinUntil(double time) { - while (GetTime() >= time) {} -} - -// Produces a random alphabet string of the specified length -static inline string RandomString(int length) { - string random_string; - for (int i = 0; i < length; i++) - random_string += rand() % 26 + 'A'; - - return random_string; -} - -// Returns a human-readable string representation of an int. -static inline string IntToString(int n) { - char s[64]; - snprintf(s, sizeof(s), "%d", n); - return string(s); -} - -// Converts a human-readable numeric string to an int. -static inline int StringToInt(const string& s) { - return atoi(s.c_str()); -} - -static inline string DoubleToString(double n) { - char s[64]; - snprintf(s, sizeof(s), "%lf", n); - return string(s); -} - -static inline double StringToDouble(const string& s) { - return atof(s.c_str()); -} - -static inline double RandomDoubleBetween(double fMin, double fMax) { - double f = (double)rand()/RAND_MAX; - return fMin + f*(fMax - fMin); -} - -// Converts a human-readable numeric sub-string (starting at the 'n'th position -// of 's') to an int. -static inline int OffsetStringToInt(const string& s, int n) { - return atoi(s.c_str() + n); -} - -// Function for deleting a heap-allocated string after it has been sent on a -// zmq socket connection. E.g., if you want to send a heap-allocated -// string '*s' on a socket 'sock': -// -// zmq::message_t msg((void*) s->data(), s->size(), DeleteString, (void*) s); -// sock.send(msg); -// -static inline void DeleteString(void* data, void* hint) { - delete reinterpret_cast(hint); -} -static inline void Noop(void* data, void* hint) {} - -//////////////////////////////// -class Mutex { - public: - // Mutexes come into the world unlocked. - Mutex() { - pthread_mutex_init(&mutex_, NULL); - } - - private: - friend class Lock; - // Actual pthread mutex wrapped by Mutex class. - pthread_mutex_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - Mutex(const Mutex&); - Mutex& operator=(const Mutex&); -}; - -class Lock { - public: - explicit Lock(Mutex* mutex) : mutex_(mutex) { - pthread_mutex_lock(&mutex_->mutex_); - } - ~Lock() { - pthread_mutex_unlock(&mutex_->mutex_); - } - - private: - Mutex* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - Lock(); - - // DISALLOW_COPY_AND_ASSIGN - Lock(const Lock&); - Lock& operator=(const Lock&); -}; - -//////////////////////////////////////////////////////////////// - -template -class AtomicQueue { - public: - AtomicQueue() { - queue_.resize(256); - size_ = 256; - front_ = 0; - back_ = 0; - } - - // Returns the number of elements currently in the queue. - inline size_t Size() { - Lock l(&size_mutex_); - return (back_ + size_ - front_) % size_; - } - - // Returns true iff the queue is empty. - inline bool Empty() { - return front_ == back_; - } - - // Atomically pushes 'item' onto the queue. - inline void Push(const T& item) { - Lock l(&back_mutex_); - // Check if the buffer has filled up. Acquire all locks and resize if so. - if (front_ == (back_+1) % size_) { - Lock m(&front_mutex_); - Lock n(&size_mutex_); - uint32 count = (back_ + size_ - front_) % size_; - queue_.resize(size_ * 2); - for (uint32 i = 0; i < count; i++) { - queue_[size_+i] = queue_[(front_ + i) % size_]; - } - front_ = size_; - back_ = size_ + count; - size_ *= 2; - } - // Push item to back of queue. - queue_[back_] = item; - back_ = (back_ + 1) % size_; - } - - // If the queue is non-empty, (atomically) sets '*result' equal to the front - // element, pops the front element from the queue, and returns true, - // otherwise returns false. - inline bool Pop(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - front_ = (front_ + 1) % size_; - return true; - } - return false; - } - - // Sets *result equal to the front element and returns true, unless the - // queue is empty, in which case does nothing and returns false. - inline bool Front(T* result) { - Lock l(&front_mutex_); - if (front_ != back_) { - *result = queue_[front_]; - return true; - } - return false; - } - - private: - vector queue_; // Circular buffer containing elements. - uint32 size_; // Allocated size of queue_, not number of elements. - uint32 front_; // Offset of first (oldest) element. - uint32 back_; // First offset following all elements. - - // Mutexes for synchronization. - Mutex front_mutex_; - Mutex back_mutex_; - Mutex size_mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicQueue(const AtomicQueue&); - AtomicQueue& operator=(const AtomicQueue&); -}; - -class MutexRW { - public: - // Mutexes come into the world unlocked. - MutexRW() { - pthread_rwlock_init(&mutex_, NULL); - } - - private: - friend class ReadLock; - friend class WriteLock; - // Actual pthread rwlock wrapped by MutexRW class. - pthread_rwlock_t mutex_; - - // DISALLOW_COPY_AND_ASSIGN - MutexRW(const MutexRW&); - MutexRW& operator=(const MutexRW&); -}; - -class ReadLock { - public: - explicit ReadLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_rdlock(&mutex_->mutex_); - } - ~ReadLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - ReadLock(); - - // DISALLOW_COPY_AND_ASSIGN - ReadLock(const ReadLock&); - ReadLock& operator=(const ReadLock&); -}; - -class WriteLock { - public: - explicit WriteLock(MutexRW* mutex) : mutex_(mutex) { - pthread_rwlock_wrlock(&mutex_->mutex_); - } - ~WriteLock() { - pthread_rwlock_unlock(&mutex_->mutex_); - } - - private: - MutexRW* mutex_; - - // DISALLOW_DEFAULT_CONSTRUCTOR - WriteLock(); - - // DISALLOW_COPY_AND_ASSIGN - WriteLock(const WriteLock&); - WriteLock& operator=(const WriteLock&); -}; - -template -class AtomicMap { - public: - AtomicMap() {} - ~AtomicMap() {} - - inline bool Lookup(const K& k, V* v) { - ReadLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup == map_.end()) { - return false; - } - *v = lookup->second; - return true; - } - - inline void Put(const K& k, const V& v) { - WriteLock l(&mutex_); - map_.insert(std::make_pair(k, v)); - } - - inline void Erase(const K& k) { - WriteLock l(&mutex_); - map_.erase(k); - } - - // Puts (k, v) if there is no record for k. Returns the value of v that is - // associated with k afterwards (either the inserted value or the one that - // was there already). - inline V PutNoClobber(const K& k, const V& v) { - WriteLock l(&mutex_); - typename unordered_map::const_iterator lookup = map_.find(k); - if (lookup != map_.end()) { - return lookup->second; - } - map_.insert(std::make_pair(k, v)); - return v; - } - - inline uint32 Size() { - ReadLock l(&mutex_); - return map_.size(); - } - - inline void DeleteVAndClear() { - WriteLock l(&mutex_); - for (typename unordered_map::iterator it = map_.begin(); - it != map_.end(); ++it) { - delete it->second; - } - map_.clear(); - } - - private: - unordered_map map_; - MutexRW mutex_; - - // DISALLOW_COPY_AND_ASSIGN - AtomicMap(const AtomicMap&); - AtomicMap& operator=(const AtomicMap&); -}; - -#endif // _DB_COMMON_UTILS_H_ - diff --git a/src_traditional_2pl_2pc_4_partitions/common/zmq.hpp b/src_traditional_2pl_2pc_4_partitions/common/zmq.hpp deleted file mode 100755 index 56653329..00000000 --- a/src_traditional_2pl_2pc_4_partitions/common/zmq.hpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2007-2011 iMatix Corporation - Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file - - This file is part of 0MQ. - - 0MQ is free software; you can redistribute it and/or modify it under - the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. - - 0MQ is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . -*/ -// Modified: Shu-Chun Weng - -#ifndef __ZMQ_HPP_INCLUDED__ -#define __ZMQ_HPP_INCLUDED__ - -#include "zmq.h" - -#include -#include -#include - -namespace zmq -{ - - typedef zmq_free_fn free_fn; - typedef zmq_pollitem_t pollitem_t; - - class error_t : public std::exception - { - public: - - error_t () : errnum (zmq_errno ()) {} - - virtual const char *what () const throw () - { - return zmq_strerror (errnum); - } - - int num () const - { - return errnum; - } - - private: - - int errnum; - }; - - inline int poll (zmq_pollitem_t *items_, int nitems_, long timeout_ = -1) - { - return zmq_poll (items_, nitems_, timeout_); - } - - inline void device (int device_, void * insocket_, void* outsocket_) - { - zmq_device (device_, insocket_, outsocket_); - } - - inline void version (int *major_, int *minor_, int *patch_) - { - zmq_version (major_, minor_, patch_); - } - - class message_t : private zmq_msg_t - { - friend class socket_t; - - public: - - inline message_t () - { - zmq_msg_init (this); - } - - inline message_t (size_t size_) - { - zmq_msg_init_size (this, size_); - } - - inline message_t (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline ~message_t () - { - int rc = zmq_msg_close (this); - assert (rc == 0); - } - - inline void rebuild () - { - zmq_msg_close (this); - zmq_msg_init (this); - } - - inline void rebuild (size_t size_) - { - zmq_msg_close (this); - zmq_msg_init_size (this, size_); - } - - inline void rebuild (void *data_, size_t size_, free_fn *ffn_, - void *hint_ = NULL) - { - zmq_msg_close (this); - zmq_msg_init_data (this, data_, size_, ffn_, hint_); - } - - inline void move (message_t *msg_) - { - zmq_msg_move (this, (zmq_msg_t*) msg_); - } - - inline void copy (message_t *msg_) - { - zmq_msg_copy (this, (zmq_msg_t*) msg_); - } - - inline void *data () - { - return zmq_msg_data (this); - } - - inline size_t size () - { - return zmq_msg_size (this); - } - - private: - - // Disable implicit message copying, so that users won't use shared - // messages (less efficient) without being aware of the fact. - message_t (const message_t&); - void operator = (const message_t&); - }; - - class context_t - { - friend class socket_t; - - public: - - inline context_t (int io_threads_) - { - ptr = zmq_init (io_threads_); - } - - inline ~context_t () - { - int rc = zmq_term (ptr); - assert (rc == 0); - } - - // Be careful with this, it's probably only useful for - // using the C api together with an existing C++ api. - // Normally you should never need to use this. - inline operator void* () - { - return ptr; - } - - private: - - void *ptr; - - context_t (const context_t&); - void operator = (const context_t&); - }; - - class socket_t - { - public: - - inline socket_t (context_t &context_, int type_) - { - ptr = zmq_socket (context_.ptr, type_); - } - - inline ~socket_t () - { - close(); - } - - inline operator void* () - { - return ptr; - } - - inline void close() - { - if(ptr == NULL) - // already closed - return ; - zmq_close (ptr); - ptr = 0 ; - } - - inline void setsockopt (int option_, const void *optval_, - size_t optvallen_) - { - zmq_setsockopt (ptr, option_, optval_, optvallen_); - } - - inline void getsockopt (int option_, void *optval_, - size_t *optvallen_) - { - zmq_getsockopt (ptr, option_, optval_, optvallen_); - } - - inline void bind (const char *addr_) - { - zmq_bind (ptr, addr_); - } - - inline void connect (const char *addr_) - { - zmq_connect (ptr, addr_); - } - - inline bool send (message_t &msg_, int flags_ = 0) - { - int rc = zmq_send (ptr, &msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - inline bool recv (message_t *msg_, int flags_ = 0) - { - int rc = zmq_recv (ptr, msg_, flags_); - if (rc == 0) - return true; - if (rc == -1 && zmq_errno () == EAGAIN) - return false; - return false; - } - - private: - - void *ptr; - - socket_t (const socket_t&); - void operator = (const socket_t&); - }; - -} - -#endif diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/.DS_Store b/src_traditional_2pl_2pc_4_partitions/deployment/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/deployment/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/deployment/Makefile.inc deleted file mode 100755 index dd50a1ba..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/Makefile.inc +++ /dev/null @@ -1,37 +0,0 @@ -OBJDIRS += deployment - -# Create template specific variables -UPPERC_DIR := DEPLOYMENT -LOWERC_DIR := deployment - -DEPLOYMENT_SRCS := -DEPLOYMENT_PROG := deployment/cluster deployment/db - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-deployment: $(DEPLOYMENT_TESTS) - @for a in $(DEPLOYMENT_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done - -# These are left over from the deployment Makefile. I'm sure there's a far -# less redundant way to incorporate these, but for now they're fine here -$(BINDIR)/deployment/cluster: $(OBJDIR)/deployment/cluster.o \ - $(OBJDIR)/common/configuration.o - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt - -$(BINDIR)/deployment/db: $(OBJDIR)/deployment/main.o \ - $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATIONS_OBJS) $(SCHEDULER_OBJS) \ - $(SEQUENCER_OBJS) $(PAXOS_OBJS) - @echo + ld $@ - @mkdir -p $(@D) - $(V)$(CXX) -o $@ $^ $(LDFLAGS) -lrt $(ZMQLDFLAGS) diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/addhosts.pl b/src_traditional_2pl_2pc_4_partitions/deployment/addhosts.pl deleted file mode 100755 index d246a3dd..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/addhosts.pl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/perl - -my @hosts; -while (<>) { - chomp; - push (@hosts, $_); - system("ssh $_ \'echo -n\'"); -} - -foreach (@hosts) { - system("scp -rp ~/.ssh/known_hosts $_:.ssh/known_hosts"); -} diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/cluster.cc b/src_traditional_2pl_2pc_4_partitions/deployment/cluster.cc deleted file mode 100755 index b6c6c7d6..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/cluster.cc +++ /dev/null @@ -1,379 +0,0 @@ -// Author: Shu-chun Weng (scweng@cs.yale.edu) -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) - - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "common/configuration.h" - -using std::map; -using std::vector; - -void ConstructDBArgs(int argc, char* argv[], int arg_begin); -bool CheckExecutable(const char* exec); -int UpdatePorts(int port_begin, Configuration* config); -void Deploy(const Configuration& config, const char* exec); - -const char default_input_config_filename[] = "deployment/test.conf"; -const char default_port_filename[] = "deployment/portfile"; -const char default_executable_filename[] = "../obj/deployment/db"; - -const char default_run_config_filename[] = "deploy-run.conf"; -const char remote_exec[] = "ssh"; - -// Command templates. - -// Redirects stdin from /dev/null (actually, prevents reading from stdin). -// This must be used when ssh is run in the background. -const char remote_opt1[] = "-nT"; -// remote_opt2 = address -const char remote_opt3_fmt[] = "cd %s; %s %d %s"; -const char remote_quite_opt3_fmt[] = "cd %s; %s %d %s > /dev/null 2>&1"; -const char remote_valgrind_opt3_fmt[] = "cd %s; valgrind %s %d %s"; -// sprintf(remote_opt3, remote_opt3_fmt, -// cwd, per-exec, node-id, db-args (joined with spaces)) - - -// TODO(scw): make deployer class; these should be class objs -char* cwd; -char* db_args; - -bool do_valgrind; -bool do_quite; - -// TODO(scw): move to deplayer class; should avoid non-POD global variable -// Type: fd -> nodeID -map children_pipes; -vector children_pids; -volatile bool end_cluster; - -int main(int argc, char* argv[]) { - int arg_begin; - - const char* config_file = default_input_config_filename; - const char* port_file = default_port_filename; - const char* exec = default_executable_filename; - - for (arg_begin = 1; arg_begin < argc; ++arg_begin) { - if (strcmp(argv[arg_begin], "-h") == 0) { - printf("Usage: %s [-v|-q] [-c config-file] [-p port-file]\n" - " [-d db-exec] [db-args..]\n" - " -c config-file default: %s\n" - " -p port-file default: %s\n" - " -d db-exec default: %s\n", - argv[0], - default_input_config_filename, - default_port_filename, - default_executable_filename); - return 0; - } else if (strcmp(argv[arg_begin], "-c") == 0) { - config_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-d") == 0) { - exec = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-p") == 0) { - port_file = argv[++arg_begin]; - assert(arg_begin < argc); - } else if (strcmp(argv[arg_begin], "-q") == 0) { - do_quite = true; - } else if (strcmp(argv[arg_begin], "-v") == 0) { - do_valgrind = true; - } else { - break; - } - } - - // filling in db_args from argv - ConstructDBArgs(argc, argv, arg_begin); - - // get a config obj - Configuration config(-1, config_file); - cwd = getcwd(NULL, 0); - - FILE* port_fp = fopen(port_file, "r"); - int port_begin; - if (port_fp == NULL) { - printf("Cannot read port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } else if (fscanf(port_fp, "%d", &port_begin) != 1) { - printf("port-file should contain a number\n"); - fclose(port_fp); - return -1; - } - fclose(port_fp); - - if (CheckExecutable(exec)) { - printf("Executable's problem\n"); - return -1; - } - - int next_port = UpdatePorts(port_begin, &config); - - port_fp = fopen(port_file, "w"); - if (port_fp == NULL) { - printf("Cannot write port file '%s': %s\n", port_file, strerror(errno)); - return -1; - } - fprintf(port_fp, "%d\n", next_port); - fclose(port_fp); - -// for (map::iterator it = config.all_nodes.begin(); -// it != config.all_nodes.end(); ++it) { -// char copy_config[1024]; -// snprintf(copy_config, sizeof(copy_config), -// "scp -rp deploy-run.conf %s:db3/deploy-run.conf", -// it->second->host.c_str()); -// system(copy_config); -// } - - Deploy(config, exec); - - delete[] db_args; - return 0; -} - -void ConstructDBArgs(int argc, char* argv[], int arg_begin) { - int len = 0; - for (int i = arg_begin; i < argc; ++i) - len += strlen(argv[i]) + 1; - db_args = new char[len + 1]; - - char* p = db_args; - for (int i = arg_begin; i < argc; ++i) - p += sprintf(p, " %s", argv[i]); -} - -bool CheckExecutable(const char* exec) { - struct stat buf; - if (stat(exec, &buf) < 0) { - printf("Cannot access %s: %s\n", exec, strerror(errno)); - return true; - } - if (!S_ISREG(buf.st_mode) || !(buf.st_mode & S_IXUSR)) { - printf("Cannot execute %s\n", exec); - return true; - } - return false; -} - -int UpdatePorts(int port_begin, Configuration* config) { - map next_port_map; - for (map::const_iterator it = config->all_nodes.begin(); - it != config->all_nodes.end(); ++it) { - Node* node = it->second; - - // Insert host, port_begin> if the host has not appeared. - // Otherwise, use the existing host-port pair. - map::iterator port_it = - next_port_map.insert(std::make_pair(node->host, port_begin)) - .first; - - node->port = port_it->second; - port_it->second = port_it->second + 1; - } - - int max_next_port = -1; - for (map::const_iterator it = next_port_map.begin(); - it != next_port_map.end(); ++it) - if (it->second > max_next_port) - max_next_port = it->second; - - return max_next_port; -} - -// deploy Node node with specified nodeid and args -void DeployOne(int nodeID, - const Node* node, - const char* exec, - const char* config_file) { - const char* remote_opt2 = node->host.c_str(); - - char copy_config[1024]; - snprintf(copy_config, sizeof(copy_config), - "scp -rp deploy-run.conf %s:db3/deploy-run.conf", - node->host.c_str()); - system(copy_config); - - char remote_opt3[1024]; - if (do_valgrind) - snprintf(remote_opt3, sizeof(remote_opt3), remote_valgrind_opt3_fmt, - cwd, exec, nodeID, db_args); - else if (do_quite) - snprintf(remote_opt3, sizeof(remote_opt3), remote_quite_opt3_fmt, - cwd, exec, nodeID, db_args); - else - snprintf(remote_opt3, sizeof(remote_opt3), remote_opt3_fmt, - cwd, exec, nodeID, db_args); - - // Black magic, don't touch (bug scw if this breaks). - int pipefd[2]; - pipe(pipefd); - int pid = fork(); - if (pid == 0) { - setsid(); - close(pipefd[0]); - dup2(pipefd[1], 1); - dup2(pipefd[1], 2); - close(pipefd[1]); - execlp("ssh", "ssh", remote_opt1, remote_opt2, remote_opt3, NULL); - printf("Node %d spawning failed\n", nodeID); - exit(-1); - } else if (pid < 0) { - printf("Node %d forking failed\n", nodeID); - } else { - children_pids.push_back(pid); - children_pipes.insert(std::pair(pipefd[0], nodeID)); - close(pipefd[1]); - } - - timespec to_sleep = { 0, 100000000 }; // 0.1 sec - nanosleep(&to_sleep, NULL); - - (void) config_file; -} - -void TerminatingChildren(int sig); -void KillRemote(const Configuration& config, const char* exec, bool client_int); -void KillLocal(); - -// deploy all nodes specified in config -void Deploy(const Configuration& config, const char* exec) { - if (!config.WriteToFile(default_run_config_filename)) { - printf("Unable to create temporary config file '%s'\n", - default_run_config_filename); - return; - } - - // use DeployOne to span components - for (map::const_iterator it = config.all_nodes.begin(); - it != config.all_nodes.end(); ++it) - DeployOne(it->first, it->second, exec, default_run_config_filename); - - // BLOCK A: Grab messages from all components, prepend node number, print. - end_cluster = false; - signal(SIGINT, &TerminatingChildren); - signal(SIGTERM, &TerminatingChildren); - signal(SIGPIPE, &TerminatingChildren); - - int num_fd = children_pipes.size(); - int max_fd = 0; - fd_set readset, fds; - FD_ZERO(&fds); - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (it->first > max_fd) - max_fd = it->first; - FD_SET(it->first, &fds); - } - ++max_fd; - - char buf[4096]; - while (num_fd > 0) { // while there are still any components alive - if (end_cluster) { - KillRemote(config, exec, false); - end_cluster = false; - } - - readset = fds; - int actions = select(max_fd, &readset, NULL, NULL, NULL); - if (actions == -1) { - if (errno == EINTR) - continue; - break; - } - - vector erasing; - for (map::const_iterator it = children_pipes.begin(); - it != children_pipes.end(); ++it) { - if (FD_ISSET(it->first, &readset)) { - int n; - if ((n = read(it->first, buf, sizeof(buf))) <= 0) { - erasing.push_back(it->first); - } else { - buf[n] = 0; - - char* save_p; - char* p = strtok_r(buf, "\n", &save_p); - do { - printf("%02d: %s\n", it->second, p); - p = strtok_r(NULL, "\n", &save_p); - } while (p); - } - } - } - if (erasing.size() > 0) { - for (vector::const_iterator it = erasing.begin(); - it != erasing.end(); ++it) { - children_pipes.erase(*it); - FD_CLR(*it, &fds); - } - num_fd -= erasing.size(); - } - } - // at the end of this while statement, either there was a user interrupt or - // no components remain alive - - // END BLOCK A - - // kill all active components & all ssh procs - KillRemote(config, exec, false); - - timespec to_sleep = { 1, 0 }; // 1 sec - nanosleep(&to_sleep, NULL); - KillLocal(); -} - -// Handles CTRL-C and other terminating signals and dies gracefully. -void TerminatingChildren(int sig) { - end_cluster = true; -} - -// try to kill all remote processes spawned -void KillRemote(const Configuration& config, - const char* exec, bool client_int) { - const char* sig_arg; - if (client_int) - sig_arg = "-INT"; - else - sig_arg = "-TERM"; - - char exec_fullpath[1024]; - snprintf(exec_fullpath, sizeof(exec_fullpath), "%s/%s", - cwd, exec); - - for (map::const_iterator it = config.all_nodes.begin() ; - it != config.all_nodes.end(); ++it) { - Node* node = it->second; - - int pid = fork(); - if (pid == 0) { - execlp("ssh", "ssh", node->host.c_str(), - "killall", sig_arg, exec_fullpath, NULL); - exit(-1); - } - } -} - -// kill all processes forked on local machine -void KillLocal() { - for (vector::const_iterator it = children_pids.begin(); - it != children_pids.end(); ++it) - kill(*it, SIGTERM); -} diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/main.cc b/src_traditional_2pl_2pc_4_partitions/deployment/main.cc deleted file mode 100755 index 13384f16..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/main.cc +++ /dev/null @@ -1,156 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Main invokation of a single node in the system. - -#include -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/simple_storage.h" -#include "scheduler/deterministic_scheduler.h" -#include "sequencer/sequencer.h" -#include "proto/tpcc_args.pb.h" - -#define HOT 100 - -unordered_map doing_deadlocks; - -// Microbenchmark load generation client. -class MClient : public Client { - public: - MClient(Configuration* config, int mp) - : microbenchmark(config->all_nodes.size(), HOT), config_(config), - percent_mp_(mp) { - } - virtual ~MClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - if (config_->all_nodes.size() > 1 && rand() % 100 < percent_mp_) { - // Multipartition txn. - int other1; - int other2; - int other3; - do { - other1 = rand() % config_->all_nodes.size(); - } while (other1 == config_->this_node_id); - - do { - other2 = rand() % config_->all_nodes.size(); - } while (other2 == config_->this_node_id || other2 == other1); - - do { - other3 = rand() % config_->all_nodes.size(); - } while (other3 == config_->this_node_id || other3 == other1 || other3 == other2); - - *txn = microbenchmark.MicroTxnMP(txn_id, config_->this_node_id, other1, other2, other3); - } else { - *txn = microbenchmark.MicroTxnSP(txn_id, config_->this_node_id); - } - } - - private: - Microbenchmark microbenchmark; - Configuration* config_; - int percent_mp_; -}; - -// TPCC load generation client. -class TClient : public Client { - public: - TClient(Configuration* config, int mp) : config_(config), percent_mp_(mp) {} - virtual ~TClient() {} - virtual void GetTxn(TxnProto** txn, int txn_id) { - TPCC tpcc; - TPCCArgs args; - - args.set_system_time(GetTime()); - if (rand() % 100 < percent_mp_) - args.set_multipartition(true); - else - args.set_multipartition(false); - - string args_string; - args.SerializeToString(&args_string); - - // New order txn - if (rand() % 100 < 100) - *txn = tpcc.NewTxn(txn_id, TPCC::NEW_ORDER, args_string, config_); - - // Payment txn - else - *txn = tpcc.NewTxn(txn_id, TPCC::PAYMENT, args_string, config_); - } - - private: - Configuration* config_; - int percent_mp_; -}; - -void stop(int sig) { -// #ifdef PAXOS -// StopZookeeper(ZOOKEEPER_CONF); -// #endif - exit(sig); -} - -int main(int argc, char** argv) { - // TODO(alex): Better arg checking. - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", - argv[0]); - exit(1); - } - - // Catch ^C and kill signals and exit gracefully (for profiling). - signal(SIGINT, &stop); - signal(SIGTERM, &stop); - - // Build this node's configuration object. - Configuration config(StringToInt(argv[1]), "deploy-run.conf"); - - // Build connection context and start multiplexer thread running. - ConnectionMultiplexer multiplexer(&config); - - // Artificial loadgen clients. - Client* client = (argv[2][0] == 'm') ? - reinterpret_cast(new MClient(&config, atoi(argv[3]))) : - reinterpret_cast(new TClient(&config, atoi(argv[3]))); - -// #ifdef PAXOS -// StartZookeeper(ZOOKEEPER_CONF); -// #endif - // Create the storage of the partition - Storage* storage = new SimpleStorage(); - if (argv[2][0] == 'm') { - Microbenchmark(config.all_nodes.size(), HOT).InitializeStorage(storage, &config); - } else { - TPCC().InitializeStorage(storage, &config); - } - - // Initialize sequencer component and start sequencer thread running. - Sequencer sequencer(&config, multiplexer.NewConnection("sequencer"), client); -storage->Initmutex(); - // Initialize scheduler component and start sequencer thread running. - if (argv[2][0] == 'm') { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new Microbenchmark(config.all_nodes.size(), HOT)); - } else { - // Run scheduler in main thread. - DeterministicScheduler scheduler(&config, - &multiplexer, - storage, - new TPCC()); - } - - Spin(120); - return 0; -} - diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/makeconf.pl b/src_traditional_2pl_2pc_4_partitions/deployment/makeconf.pl deleted file mode 100755 index ff5fb1a2..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/makeconf.pl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/perl -# -# This program reads a newline-delimited list of ip addresses from stdin and -# prints a valid config file to stdout whose consecutively-numbered nodes have -# the specified ip addresses as their hostnames. - -my $i = 0; -while (<>) { - chomp; - print "node$i=0:$i:8:$_:50000\n"; - $i++; -} - diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/portfile b/src_traditional_2pl_2pc_4_partitions/deployment/portfile deleted file mode 100755 index 57d37c4c..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/portfile +++ /dev/null @@ -1 +0,0 @@ -53527 diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/run.pbs b/src_traditional_2pl_2pc_4_partitions/deployment/run.pbs deleted file mode 100755 index 178caac7..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/run.pbs +++ /dev/null @@ -1,38 +0,0 @@ -#PBS -q default -#PBS -lnodes=3:ppn=8 -#PBS -l walltime=4:00:00 -#PBS -m abe -M thomson@cs.yale.edu - -cd db3 - -# set up test.conf and client_host -deployment/makeconf.pl > deployment/test.conf - -# start system running -../obj/deployment/cluster deployment/test.conf deployment/portfile ../obj/deployment/db & - -# after brief delay, start clients running -sleep 5 -perl <; - my @nodes = unique(@allnodes); - for(my \$i = 0; \$i <= \$#nodes; \$i++) { - my \$addr = \$nodes[\$i]; - chomp \$addr; - system("ssh \$addr \"cd db3; ../obj/deployment/client m 0\""); - } - close(NODES); -END - diff --git a/src_traditional_2pl_2pc_4_partitions/deployment/test.conf b/src_traditional_2pl_2pc_4_partitions/deployment/test.conf deleted file mode 100755 index 108bea6e..00000000 --- a/src_traditional_2pl_2pc_4_partitions/deployment/test.conf +++ /dev/null @@ -1,339 +0,0 @@ -# Node=:::: - - -node0=0:0:12:128.36.232.18:51262 # grizzly -node1=0:0:12:128.36.232.12:51261 #rhino -#node2=0:0:12:128.36.232.15:51262 # gator -#node3=0:0:12:128.36.232.20:51261 #macaw -#node4=0:0:12:128.36.232.19:51262 # peacock -#node5=0:0:12:128.36.232.34:51261 # perch -#node6=0:0:12:128.36.232.14:51262 # jaguar -#node7=0:0:12:128.36.232.16:51261 # hippo -#node8=0:0:12:128.36.232.39:51261 #viper -#node9=0:0:12:128.36.232.7:51261 #hornet -#node10=0:0:12:128.36.232.8:51261 #cricket -#node11=0:0:12:128.36.232.28:51261 #termite -#node12=0:0:12:128.36.232.5:51261 #cicada -#node13=0:0:12:128.36.232.6:51261 #bumblebee -#node14=0:0:12:128.36.232.44:51261 #frog -#node15=0:0:12:128.36.232.50:51261 #tick -#node16=0:0:12:128.36.232.42:51261 #turtle -#node17=0:0:12:128.36.232.35:51261 #aphid -#node18=0:0:12:128.36.232.41:51261 #chameleon -#node19=0:0:12:128.36.232.38:51261 #rattlesnake -#node20=0:0:12:128.36.232.45:51261 #scorpion -#node21=0:0:12:128.36.232.29:51261 #cardinal -#node22=0:0:12:128.36.232.24:51261 #swan -#node23=0:0:12:128.36.232.27:51261 #monkey - - - -#node2=0:8:16:128.36.232.16:51471 # lion -#node2=0:9:16:128.36.232.16:50005 -#node2=0:10:16:128.36.232.16:50005 -#node2=0:11:16:128.36.232.16:50005 -#node3=0:12:16:128.36.232.20:51472 -#node3=0:13:16:128.36.232.20:51473 -#node3=0:14:16:128.36.232.20:51473 -#node3=0:15:16:128.36.232.20:51473 - - - - - -#node0=0:0:16:128.36.232.9:51262 # lion -#node0=0:1:16:128.36.232.9:50005 -#node0=0:2:16:128.36.232.9:50005 -#node0=0:3:16:128.36.232.9:50005 -#node1=0:4:16:128.36.232.14:51471 # jaguar -#node1=0:5:16:128.36.232.14:50005 -#node1=0:6:16:128.36.232.14:50005 -#node1=0:7:16:128.36.232.14:50005 -#node2=0:8:16:128.36.232.15:51471 # gator -#node2=0:9:16:128.36.232.15:50005 -#node2=0:10:16:128.36.232.15:50005 -#node2=0:11:16:128.36.232.15:50005 -#node3=0:12:16:128.36.232.18:51471 # grizzly -#node3=0:13:16:128.36.232.18:50005 -#node3=0:14:16:128.36.232.18:50005 -#node3=0:15:16:128.36.232.18:50005 -#node4=0:16:16:128.36.232.16:51471 # hippo -#node4=0:17:16:128.36.232.16:50005 -#node4=0:18:16:128.36.232.16:50005 -#node4=0:19:16:128.36.232.16:50005 -#node5=0:20:16:128.36.232.34:51471 # perch -#node5=0:21:16:128.36.232.34:50005 -#node5=0:22:16:128.36.232.34:50005 -#node5=0:23:16:128.36.232.34:50005 -#node6=0:24:16:128.36.232.37:51471 # cobra -#node6=0:25:16:128.36.232.37:50005 -#node6=0:26:16:128.36.232.37:50005 -#node6=0:27:16:128.36.232.37:50005 -#node7=0:28:16:128.36.232.12:51471 # rhino -#node7=0:29:16:128.36.232.12:50005 -#node7=0:30:16:128.36.232.12:50005 -#node7=0:31:16:128.36.232.12:50005 -#node8=0:32:16:128.36.232.8:50005 # cricket -#node8=0:33:16:128.36.232.8:50005 # cricket -#node8=0:34:16:128.36.232.8:50005 # cricket -#node8=0:35:16:128.36.232.8:50005 # cricket -#node9=0:36:16:128.36.232.43:51471 # newt -#node9=0:37:16:128.36.232.43:50005 -#node9=0:38:16:128.36.232.43:50005 -#node9=0:39:16:128.36.232.43:50005 -#node10=0:40:16:128.36.232.45:51471 # scorpion -#node10=0:41:16:128.36.232.45:50005 -#node10=0:42:16:128.36.232.45:50005 -#node10=0:43:16:128.36.232.45:50005 -#node11=0:44:16:128.36.232.29:51471 # cardinal -#node11=0:45:16:128.36.232.29:50005 -#node11=0:46:16:128.36.232.29:50005 -#node11=0:47:16:128.36.232.29:50005 -#node12=0:48:16:128.36.232.27:51471 # monkey -#node12=0:49:16:128.36.232.27:50005 -#node12=0:50:16:128.36.232.27:50005 -#node12=0:51:16:128.36.232.27:50005 -#node13=0:52:16:128.36.232.39:51471 # viper -#node13=0:53:16:128.36.232.39:50005 -#node13=0:54:16:128.36.232.39:50005 -#node13=0:55:16:128.36.232.39:50005 -#node14=0:56:16:128.36.232.7:51471 # hornet -#node14=0:57:16:128.36.232.7:50005 -#node14=0:58:16:128.36.232.7:50005 -#node14=0:59:16:128.36.232.7:50005 -#node15=0:60:16:128.36.232.28:50005 # termite -#node15=0:61:16:128.36.232.28:50005 # termite -#node15=0:62:16:128.36.232.28:50005 # termite -#node15=0:63:16:128.36.232.28:50005 # termite -#node16=0:64:16:128.36.232.50:50005 # tick -#node16=0:65:16:128.36.232.50:50005 # tick -#node16=0:66:16:128.36.232.50:50005 # tick -#node16=0:67:16:128.36.232.50:50005 # tick -#node17=0:68:16:128.36.232.20:51471 # macaw -#node17=0:69:16:128.36.232.20:50005 -#node17=0:70:16:128.36.232.20:50005 -#node17=0:71:16:128.36.232.20:50005 -#node18=0:72:16:128.36.232.44:50004 # frog -#node18=0:73:16:128.36.232.44:50004 # frog -#node18=0:74:16:128.36.232.44:50004 # frog -#node18=0:75:16:128.36.232.44:50004 # frog -#node19=0:76:16:128.36.232.42:50003 # turtle -#node19=0:77:16:128.36.232.42:50003 # turtle -#node19=0:78:16:128.36.232.42:50003 # turtle -#node19=0:79:16:128.36.232.42:50003 # turtle - - -#node0=0:0:16:128.36.232.16:51262 # hippo -#node0=0:1:16:128.36.232.16:50005 -#node0=0:2:16:128.36.232.16:50005 -#node0=0:3:16:128.36.232.16:50005 -#node1=0:4:16:128.36.232.29:51471 # cardinal -#node1=0:5:16:128.36.232.29:50005 -#node1=0:6:16:128.36.232.29:50005 -#node1=0:7:16:128.36.232.29:50005 -#node2=0:8:16:128.36.232.6:51471 # bumblebee -#node2=0:9:16:128.36.232.6:50005 -#node2=0:10:16:128.36.232.6:50005 -#node2=0:11:16:128.36.232.6:50005 -#node3=0:12:16:128.36.232.18:51471 # grizzly -#node3=0:13:16:128.36.232.18:50005 -#node3=0:14:16:128.36.232.18:50005 -#node3=0:15:16:128.36.232.18:50005 -#node4=0:16:16:128.36.232.15:51471 # gator -#node4=0:17:16:128.36.232.15:50005 -#node4=0:18:16:128.36.232.15:50005 -#node4=0:19:16:128.36.232.15:50005 -#node5=0:20:16:128.36.232.38:51471 # rattlesnake -#node5=0:21:16:128.36.232.38:50005 -#node5=0:22:16:128.36.232.38:50005 -#node5=0:23:16:128.36.232.38:50005 -#node6=0:24:16:128.36.232.28:51471 # termite -#node6=0:25:16:128.36.232.28:50005 -#node6=0:26:16:128.36.232.28:50005 -#node6=0:27:16:128.36.232.28:50005 -#node7=0:28:16:128.36.232.34:51471 # perch -#node7=0:29:16:128.36.232.34:50005 -#node7=0:30:16:128.36.232.34:50005 -#node7=0:31:16:128.36.232.34:50005 -#node8=0:32:16:128.36.232.20:51471 # macaw -#node8=0:33:16:128.36.232.20:50005 -#node8=0:34:16:128.36.232.20:50005 -#node8=0:35:16:128.36.232.20:50005 -#node9=0:36:16:128.36.232.14:51471 # jaguar -#node9=0:37:16:128.36.232.14:50005 -#node9=0:38:16:128.36.232.14:50005 -#node9=0:39:16:128.36.232.14:50005 -#node10=0:40:16:128.36.232.7:51471 # hornet -#node10=0:41:16:128.36.232.7:50005 -#node10=0:42:16:128.36.232.7:50005 -#node10=0:43:16:128.36.232.7:50005 -#node11=0:44:16:128.36.232.27:51471 # monkey -#node11=0:45:16:128.36.232.27:50005 -#node11=0:46:16:128.36.232.27:50005 -#node11=0:47:16:128.36.232.27:50005 -#node12=0:48:16:128.36.232.43:51471 # newt -#node12=0:49:16:128.36.232.43:50005 -#node12=0:50:16:128.36.232.43:50005 -#node12=0:51:16:128.36.232.43:50005 -#node13=0:52:16:128.36.232.35:51471 # aphid -#node13=0:53:16:128.36.232.35:50005 -#node13=0:54:16:128.36.232.35:50005 -#node13=0:55:16:128.36.232.35:50005 -#node14=0:56:16:128.36.232.37:51471 # cobra -#node14=0:57:16:128.36.232.37:50005 -#node14=0:58:16:128.36.232.37:50005 -#node14=0:59:16:128.36.232.37:50005 -#node15=0:60:16:128.36.232.45:51471 # scorpion -#node15=0:61:16:128.36.232.45:50005 -#node15=0:62:16:128.36.232.45:50005 -#node15=0:63:16:128.36.232.45:50005 - - -#node0=0:0:16:128.36.232.7:51471 # hornet -#node0=0:1:16:128.36.232.7:50005 -#node0=0:2:16:128.36.232.7:50005 -#node0=0:3:16:128.36.232.7:50005 -#node1=0:4:16:128.36.232.27:51471 #monkey -#node1=0:5:16:128.36.232.27:50005 -#node1=0:6:16:128.36.232.27:50005 -#node1=0:7:16:128.36.232.27:50005 -#node2=0:8:16:128.36.232.8:51471 #cricket -#node2=0:9:16:128.36.232.8:50005 -#node2=0:10:16:128.36.232.8:50005 -#node2=0:11:16:128.36.232.8:50005 -#node3=0:12:16:128.36.232.15:51471 # gator -#node3=0:13:16:128.36.232.15:50005 -#node3=0:14:16:128.36.232.15:50005 -#node3=0:15:16:128.36.232.15:50005 -#node4=0:16:16:128.36.232.6:51471 # bumblebee -#node4=0:17:16:128.36.232.6:50005 -#node4=0:18:16:128.36.232.6:50005 -#node4=0:19:16:128.36.232.6:50005 -#node5=0:20:16:128.36.232.41:51471 # chameleon -#node5=0:21:16:128.36.232.41:50005 -#node5=0:22:16:128.36.232.41:50005 -#node5=0:23:16:128.36.232.41:50005 -#node6=0:24:16:128.36.232.49:51471 # ladybug -#node6=0:25:16:128.36.232.49:50005 -#node6=0:26:16:128.36.232.49:50005 -#node6=0:27:16:128.36.232.49:50005 -#node7=0:28:16:128.36.232.43:51471 # newt -#node7=0:29:16:128.36.232.43:50005 -#node7=0:30:16:128.36.232.43:50005 -#node7=0:31:16:128.36.232.43:50005 -#node8=0:32:16:128.36.232.39:51471 # viper -#node8=0:33:16:128.36.232.39:50005 -#node8=0:34:16:128.36.232.39:50005 -#node8=0:35:16:128.36.232.39:50005 -#node9=0:36:16:128.36.232.5:51471 # cicada -#node9=0:37:16:128.36.232.5:50005 -#node9=0:38:16:128.36.232.5:50005 -#node9=0:39:16:128.36.232.5:50005 -#node10=0:40:16:128.36.232.38:51471 # rattlesnake -#node10=0:41:16:128.36.232.38:50005 -#node10=0:42:16:128.36.232.38:50005 -#node10=0:43:16:128.36.232.38:50005 -#node11=0:44:16:128.36.232.29:51471 # cardinal -#node11=0:45:16:128.36.232.29:50005 -#node11=0:46:16:128.36.232.29:50005 -#node11=0:47:16:128.36.232.29:50005 -#node12=0:48:16:128.36.232.16:51471 # hippo -#node12=0:49:16:128.36.232.16:50005 -#node12=0:50:16:128.36.232.16:50005 -#node12=0:51:16:128.36.232.16:50005 -#node13=0:52:16:128.36.232.12:51471 # rhino -#node13=0:53:16:128.36.232.12:50005 -#node13=0:54:16:128.36.232.12:50005 -#node13=0:55:16:128.36.232.12:50005 -#node14=0:56:16:128.36.232.14:51471 # jaguar -#node14=0:57:16:128.36.232.14:50005 -#node14=0:58:16:128.36.232.14:50005 -#node14=0:59:16:128.36.232.14:50005 -#node15=0:60:16:128.36.232.20:51471 # macaw -#node15=0:61:16:128.36.232.20:50005 -#node15=0:62:16:128.36.232.20:50005 -#node15=0:63:16:128.36.232.20:50005 - -#node0=0:0:16:128.36.232.8:50005 # cricket -#node0=0:1:16:128.36.232.8:50005 # cricket -#node0=0:2:16:128.36.232.8:50005 # cricket -#node0=0:3:16:128.36.232.8:50005 # cricket -#node1=0:4:16:128.36.232.6:50005 # bumblebee -#node1=0:5:16:128.36.232.6:50005 # bumblebee -#node1=0:6:16:128.36.232.6:50005 # bumblebee -#node1=0:7:16:128.36.232.6:50005 # bumblebee -#node2=0:8:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:9:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:10:16:128.36.232.38:50005 # lion//rattlesnake -#node2=0:11:16:128.36.232.38:50005 # lion//rattlesnake -#node3=0:12:16:128.36.232.34:50001 # perch -#node3=0:13:16:128.36.232.34:50001 # perch -#node3=0:14:16:128.36.232.34:50001 # perch -#node3=0:15:16:128.36.232.34:50001 # perch -#node4=0:16:16:128.36.232.39:50002 # viper -#node4=0:17:16:128.36.232.39:50002 # viper -#node4=0:18:16:128.36.232.39:50002 # viper -#node4=0:19:16:128.36.232.39:50002 # viper -#node5=0:20:16:128.36.232.42:50003 # turtle -#node5=0:21:16:128.36.232.42:50003 # turtle -#node5=0:22:16:128.36.232.42:50003 # turtle -#node5=0:23:16:128.36.232.42:50003 # turtle -#node6=0:24:16:128.36.232.44:50004 # frog -#node6=0:25:16:128.36.232.44:50004 # frog -#node6=0:26:16:128.36.232.44:50004 # frog -#node6=0:27:16:128.36.232.44:50004 # frog -#node7=0:28:16:128.36.232.41:50005 # chameleon -#node7=0:29:16:128.36.232.41:50005 # chameleon -#node7=0:30:16:128.36.232.41:50005 # chameleon -#node7=0:31:16:128.36.232.41:50005 # chameleon -#node8=0:32:16:128.36.232.45:50005 # scorpion -#node8=0:33:16:128.36.232.45:50005 # scorpion -#node8=0:34:16:128.36.232.45:50005 # scorpion -#node8=0:35:16:128.36.232.45:50005 # scorpion -#node9=0:36:16:128.36.232.49:50005 # ladybug -#node9=0:37:16:128.36.232.49:50005 # ladybug -#node9=0:38:16:128.36.232.49:50005 # ladybug -#node9=0:39:16:128.36.232.49:50005 # ladybug -#node10=0:40:16:128.36.232.7:50005 # hornet -#node10=0:41:16:128.36.232.7:50005 # hornet -#node10=0:42:16:128.36.232.7:50005 # hornet -#node10=0:43:16:128.36.232.7:50005 # hornet -#node11=0:44:16:128.36.232.43:50005 # newt -#node11=0:45:16:128.36.232.43:50005 # newt -#node11=0:46:16:128.36.232.43:50005 # newt -#node11=0:47:16:128.36.232.43:50005 # newt -#node12=0:48:16:128.36.232.28:50005 # termite -#node12=0:49:16:128.36.232.28:50005 # termite -#node12=0:50:16:128.36.232.28:50005 # termite -#node12=0:51:16:128.36.232.28:50005 # termite -#node13=0:52:16:128.36.232.46:50005 # python -#node13=0:53:16:128.36.232.46:50005 # python -#node13=0:54:16:128.36.232.46:50005 # python -#node13=0:55:16:128.36.232.46:50005 # python -#node14=0:56:16:128.36.232.5:50005 # cicada -#node14=0:57:16:128.36.232.5:50005 # cicada -#node14=0:58:16:128.36.232.5:50005 # cicada -#node14=0:59:16:128.36.232.5:50005 # cicada -#node15=0:60:16:128.36.232.35:50005 # aphid -#node15=0:61:16:128.36.232.35:50005 # aphid -#node15=0:62:16:128.36.232.35:50005 # aphid -#node15=0:63:16:128.36.232.35:50005 # aphid - - - - - - - - - - - -#node0=0:0:16:128.36.232.37:50001 # cobra -#node1=0:1:16:128.36.232.39:50002 # viper -#node2=0:2:16:128.36.232.42:50003 # turtle -#node3=0:3:16:128.36.232.44:50004 # frog -#node4=0:4:16:128.36.232.50:50005 # tick -#node5=0:5:16:128.36.232.8:50005 # cricket -#node6=0:6:16:128.36.232.9:50005 # lion -#node7=0:7:16:128.36.232.6:50005 # bumblebee diff --git a/src_traditional_2pl_2pc_4_partitions/obj/.deps b/src_traditional_2pl_2pc_4_partitions/obj/.deps deleted file mode 100755 index e69de29b..00000000 diff --git a/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.d b/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.d deleted file mode 100755 index d4246650..00000000 --- a/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.d +++ /dev/null @@ -1,90 +0,0 @@ -../obj/common/configuration.o: common/configuration.cc \ - ../src/common/configuration.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdint.h \ - /usr/include/stdint.h /usr/include/features.h /usr/include/sys/cdefs.h \ - /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ - /usr/include/gnu/stubs-64.h /usr/include/bits/wchar.h \ - /usr/include/c++/4.6/map /usr/include/c++/4.6/bits/stl_tree.h \ - /usr/include/c++/4.6/bits/stl_algobase.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++config.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/os_defines.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/cpu_defines.h \ - /usr/include/c++/4.6/bits/functexcept.h \ - /usr/include/c++/4.6/bits/exception_defines.h \ - /usr/include/c++/4.6/bits/cpp_type_traits.h \ - /usr/include/c++/4.6/ext/type_traits.h \ - /usr/include/c++/4.6/ext/numeric_traits.h \ - /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ - /usr/include/c++/4.6/bits/concept_check.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ - /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ - /usr/include/c++/4.6/bits/stl_iterator.h \ - /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/allocator.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++allocator.h \ - /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ - /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/stl_function.h \ - /usr/include/c++/4.6/backward/binders.h \ - /usr/include/c++/4.6/bits/stl_map.h \ - /usr/include/c++/4.6/initializer_list \ - /usr/include/c++/4.6/bits/stl_multimap.h \ - /usr/include/c++/4.6/bits/range_access.h /usr/include/c++/4.6/string \ - /usr/include/c++/4.6/bits/stringfwd.h \ - /usr/include/c++/4.6/bits/char_traits.h \ - /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ - /usr/include/wchar.h /usr/include/stdio.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stdarg.h \ - /usr/lib64/gcc/x86_64-suse-linux/4.6/include/stddef.h \ - /usr/include/xlocale.h /usr/include/c++/4.6/bits/localefwd.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/c++locale.h \ - /usr/include/c++/4.6/clocale /usr/include/locale.h \ - /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ - /usr/include/c++/4.6/cctype /usr/include/ctype.h \ - /usr/include/bits/types.h /usr/include/bits/typesizes.h \ - /usr/include/endian.h /usr/include/bits/endian.h \ - /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ - /usr/include/c++/4.6/bits/cxxabi_forced.h \ - /usr/include/c++/4.6/bits/basic_string.h \ - /usr/include/c++/4.6/ext/atomicity.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/gthr-default.h \ - /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ - /usr/include/bits/sched.h /usr/include/bits/time.h \ - /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ - /usr/include/bits/setjmp.h /usr/include/unistd.h \ - /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ - /usr/include/bits/confname.h /usr/include/getopt.h \ - /usr/include/c++/4.6/x86_64-suse-linux/bits/atomic_word.h \ - /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/vector \ - /usr/include/c++/4.6/bits/stl_construct.h \ - /usr/include/c++/4.6/bits/stl_uninitialized.h \ - /usr/include/c++/4.6/bits/stl_vector.h \ - /usr/include/c++/4.6/bits/stl_bvector.h \ - /usr/include/c++/4.6/bits/vector.tcc \ - /usr/include/c++/4.6/tr1/unordered_map /usr/include/c++/4.6/utility \ - /usr/include/c++/4.6/bits/stl_relops.h \ - /usr/include/c++/4.6/tr1/type_traits \ - /usr/include/c++/4.6/tr1/functional_hash.h \ - /usr/include/c++/4.6/tr1/hashtable.h \ - /usr/include/c++/4.6/tr1/hashtable_policy.h \ - /usr/include/c++/4.6/tr1/unordered_map.h ../src/common/types.h \ - /usr/include/netdb.h /usr/include/netinet/in.h /usr/include/sys/socket.h \ - /usr/include/sys/uio.h /usr/include/sys/types.h \ - /usr/include/sys/select.h /usr/include/bits/select.h \ - /usr/include/bits/sigset.h /usr/include/sys/sysmacros.h \ - /usr/include/bits/uio.h /usr/include/bits/socket.h \ - /usr/include/bits/sockaddr.h /usr/include/asm/socket.h \ - /usr/include/asm-generic/socket.h /usr/include/asm/sockios.h \ - /usr/include/asm-generic/sockios.h /usr/include/bits/in.h \ - /usr/include/rpc/netdb.h /usr/include/bits/siginfo.h \ - /usr/include/bits/netdb.h /usr/include/c++/4.6/cstdio \ - /usr/include/libio.h /usr/include/_G_config.h \ - /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ - /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ - /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ - /usr/include/alloca.h /usr/include/c++/4.6/cstring /usr/include/string.h \ - ../src/common/utils.h /usr/include/assert.h /usr/include/sys/time.h \ - /usr/include/c++/4.6/cmath /usr/include/math.h \ - /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ - /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ - /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ - /usr/include/bits/mathcalls.h diff --git a/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.o b/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.o deleted file mode 100755 index 510cfb78..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/obj/common/configuration.o and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/paxos/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/paxos/Makefile.inc deleted file mode 100755 index 45d9bdd2..00000000 --- a/src_traditional_2pl_2pc_4_partitions/paxos/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += paxos - -# Create template specific variables -UPPERC_DIR := PAXOS -LOWERC_DIR := paxos - -PAXOS_PROG := -PAXOS_SRCS := paxos/paxos.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-paxos: $(PAXOS_TESTS) - @for a in $(PAXOS_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/paxos/paxos.cc b/src_traditional_2pl_2pc_4_partitions/paxos/paxos.cc deleted file mode 100755 index d0523e10..00000000 --- a/src_traditional_2pl_2pc_4_partitions/paxos/paxos.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#include "paxos/paxos.h" - -#include -#include -#include - -using std::ifstream; -using std::pair; -using std::vector; - -Paxos::Paxos(const string& zookeeper_config_file, bool reader) { - ifstream in(zookeeper_config_file.c_str()); - string s, port, ip, connection_string, timeout; - // Get the connection string(ip and port) from the config file. - while (getline(in, s)) { - if (s.substr(0, 10) == "clientPort") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - port = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "server1") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - ip = s.substr(pos1+1, pos2-pos1-1); - } else if (s.substr(0, 7) == "timeout") { - int pos1 = s.find('='); - int pos2 = s.find('\0'); - timeout = s.substr(pos1+1, pos2-pos1-1); - } - } - connection_string = ip + ":" + port; - - next_read_batch_index_ = 0; - - // Init the mutexes. - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_init(&(mutexes_[i]), NULL); - } - - // Connect to the zookeeper. - zh_ = zookeeper_init(connection_string.c_str(), NULL, - atoi(timeout.c_str()), 0, NULL, 0); - if (zh_ == NULL) { - printf("Connection to zookeeper failed.\n"); - return; - } - - // Verify that whether the root node have been created, - // if not, create the root node. - int rc = zoo_exists(zh_, "/root", 0, NULL); - if (rc == ZNONODE) { - // If multiple nodes executing this code to both see that - // the root doesn't exist, only one node creates /root - // node actually, the others return ZNODEEXISTS. - int create_rc = zoo_create(zh_, "/root", NULL, 0, - &ZOO_OPEN_ACL_UNSAFE, - 0, NULL, 0); - if (create_rc != ZOK && create_rc != ZNODEEXISTS) { - printf("zoo_create error:error number is %d\n", create_rc); - } - } - - // Get batches from the zookeeper concurrently if 'reader' is set to true. - if (reader) { - for (uint64 i = 0; i < CONCURRENT_GETS; i++) { - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", i); - int get_rc = zoo_aget(zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast( - new pair< uint64, Paxos*>(i, this))); - if (get_rc) { - printf("Have exited the Paxos thread, exit number is %d.\n", get_rc); - } - } - } -} - -Paxos::~Paxos() { - // Destroy the mutexes. - for (uint i = 0; i < CONCURRENT_GETS; i++) { - pthread_mutex_destroy(&(mutexes_[i])); - } - // Close the connection with the zookeeper. - int rc = zookeeper_close(zh_); - if (rc != ZOK) { - printf("zookeeper_close error:error number is %d\n", rc); - } -} - -void Paxos::SubmitBatch(const string& batch_data) { - // Submit batch means that create new znode below the root directory. - int rc = zoo_acreate(zh_, "/root/batch-", batch_data.c_str(), - batch_data.size(), &ZOO_OPEN_ACL_UNSAFE, - ZOO_SEQUENCE | ZOO_EPHEMERAL, - acreate_completion, NULL); - if (rc != ZOK) { - printf("zoo_acreate error:error number is %d\n", rc); - } -} - -bool Paxos::GetNextBatch(string* batch_data) { - int next_batch_thread = next_read_batch_index_ % CONCURRENT_GETS; - // If there have been some batches stored in the corresponding batch_table, - // read from that and return true, else return false. - if (batch_tables_[next_batch_thread].size() > 0) { - // Lock the batch table. - pthread_mutex_lock(&(mutexes_[next_batch_thread])); - (*batch_data) = batch_tables_[next_batch_thread][next_read_batch_index_]; - batch_tables_[next_batch_thread].erase(next_read_batch_index_); - // Unlock the batch table. - pthread_mutex_unlock(&(mutexes_[next_batch_thread])); - next_read_batch_index_++; - return true; - } else { - return false; - } -} - -void Paxos::GetNextBatchBlocking(string* batch_data) { - while (!GetNextBatch(batch_data)) { - } -} - -void Paxos::get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data) { - // XXX(scw): using const_cast is disgusting - pair* previous_data = - reinterpret_cast*>(const_cast(data)); - uint64 previous_index_for_aget = previous_data->first; - Paxos* paxos = previous_data->second; - string batch_data(value, value_len); - uint64 next_index_for_aget; - // If zoo_aget function completed successfully, insert the batch into the - // corresponding batch_tables_. - if (rc == ZOK) { - // Set the number of batch which will be got from zookeeper next time - // (just plus the CONCURRENT_GETS). - next_index_for_aget = previous_index_for_aget + CONCURRENT_GETS; - pthread_mutex_lock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - paxos->batch_tables_[previous_index_for_aget % CONCURRENT_GETS] - [previous_index_for_aget] = batch_data; - pthread_mutex_unlock(&paxos->mutexes_[previous_index_for_aget % - CONCURRENT_GETS]); - // If there are no new batch in the zookeeper, just wait for a while - // and continue to get from zookeeper. - } else if (rc == ZNONODE) { - next_index_for_aget = previous_index_for_aget; - usleep(0.2*1000); - } else { - return; - } - // Continue to get a batch from zookeeper. - char current_read_batch_path[23]; - snprintf(current_read_batch_path, sizeof(current_read_batch_path), - "%s%010lu", "/root/batch-", next_index_for_aget); - previous_data->first = next_index_for_aget; - int get_rc = zoo_aget(paxos->zh_, current_read_batch_path, 0, - get_data_completion, - reinterpret_cast(previous_data)); - if (get_rc) { - return; - } -} - -void Paxos::acreate_completion(int rc, const char *name, const void * data) { - if (rc) { - printf("Error %d for zoo_acreate.\n", rc); - } -} - -// This function will automatically start zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StartZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - } - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - string ssh_command = "ssh " + zookeepers[i] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/" + - "bin/zkServer.sh start > zookeeper_log &"; - // Run the ssh command. - system(ssh_command.c_str()); - } - printf("Starting zookeeper servers.\n"); - sleep(8); -} - -// This function will automatically stop zookeeper server based on the -// zookeeper config file(generate ssh commands and execute them). -void StopZookeeper(const string& zookeeper_config_file) { - vector zookeepers; - string line , port, ssh_command; - // Read zookeeper config file. - ifstream in(zookeeper_config_file.c_str()); - // Put all zookeeper server's ip into the vector. - while (getline(in, line)) { - if (line.substr(0, 6) == "server") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - zookeepers.push_back(line.substr(pos1+1, pos2-pos1-1)); - } - if (line.substr(0, 10) == "clientPort") { - int pos1 = line.find('='); - int pos2 = line.find('\0'); - port = line.substr(pos1+1, pos2-pos1-1); - } - } - ssh_command = "ssh " + zookeepers[0] + - " /tmp/kr358/zookeeper/zookeeper-3.3.3/bin/zkCli.sh -server " - + zookeepers[0] + ":" + port + " delete /root > zookeeper_log"; - system(ssh_command.c_str()); - sleep(2); - for (unsigned int i = 0; i< zookeepers.size(); i++) { - // Generate the ssh command. - ssh_command = "ssh " + zookeepers[i] + " /tmp/kr358/zookeeper/" - + "zookeeper-3.3.3/bin/zkServer.sh stop > zookeeper_log &"; - system(ssh_command.c_str()); - } -} - diff --git a/src_traditional_2pl_2pc_4_partitions/paxos/paxos.h b/src_traditional_2pl_2pc_4_partitions/paxos/paxos.h deleted file mode 100755 index 8488953e..00000000 --- a/src_traditional_2pl_2pc_4_partitions/paxos/paxos.h +++ /dev/null @@ -1,78 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) -// Alexander Thomson (thomson@cs.yale.edu) -// The Paxos object allows batches to be registered with a running zookeeper -// instance, inserting them into a globally consistent batch order. - -#ifndef _DB_PAXOS_PAXOS_H_ -#define _DB_PAXOS_PAXOS_H_ - -#include -#include -#include -#include - -#include "common/types.h" - -using std::map; -using std::string; - -// The path of zookeeper config file -#define ZOOKEEPER_CONF "paxos/zookeeper.conf" - -// Number of concurrently get batches from the zookeeper servers -// at any given time. -#define CONCURRENT_GETS 128 - -class Paxos { - public: - // Construct and initialize a Paxos object. Configuration of the associated - // zookeeper instance is read from the file whose path is identified by - // 'zookeeper_conf_file'. If 'reader' is not set to true, GetNextBatch may - // never be called on this Paxos object. - Paxos(const string& zookeeper_config_file, bool reader); - - // Deconstructor closes the connection with the zookeeper service. - ~Paxos(); - - // Sends a new batch to the associated zookeeper instance. Does NOT block. - // The zookeeper service will create a new znode whose data is 'batch_data', - // thus inserting the batch into the global order. Once a quorum of zookeeper - // nodes have agreed on an insertion, it will appear in the same place in - // the global order to all readers. - void SubmitBatch(const string& batch_data); - - // Attempts to read the next batch in the global sequence into '*batch_data'. - // Returns true on successful read of the next batch. Like SubmitBatch, - // GetNextBatch does NOT block if the next batch is not immediately known, - // but rather returns false immediately. - bool GetNextBatch(string* batch_data); - - // Reads the next batch in the global sequence into '*batch_data'. If it is - // not immediately known, GetNextBatchBlocking blocks until it is received. - void GetNextBatchBlocking(string* batch_data); - - private: - // The zookeeper handle obtained by a call to zookeeper_init. - zhandle_t *zh_; - - // Record the serial number of the batch which will be read next time. - uint64 next_read_batch_index_; - - // The mutex lock of every concurrent get(because the map container - // is not thread safe). - pthread_mutex_t mutexes_[CONCURRENT_GETS]; - - // The map array save the batches which are concurrently got from zookeeper. - map batch_tables_[CONCURRENT_GETS]; - - // For zoo_aget completion function, this method will be invoked - // at the end of a asynchronous call( zoo_aget is asynchronous call - // which get data from zookeeper). - static void get_data_completion(int rc, const char *value, int value_len, - const struct Stat *stat, const void *data); - - // For zoo_acreate completion function, this method will be invoked - // at the end of zoo_acreate function. - static void acreate_completion(int rc, const char *name, const void * data); -}; -#endif // _DB_PAXOS_PAXOS_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/paxos/zookeeper.conf b/src_traditional_2pl_2pc_4_partitions/paxos/zookeeper.conf deleted file mode 100755 index f4123548..00000000 --- a/src_traditional_2pl_2pc_4_partitions/paxos/zookeeper.conf +++ /dev/null @@ -1,5 +0,0 @@ -clientPort=2181 -server1=128.36.232.9 -server2=128.36.232.50 -server3=128.36.232.44 -timeout=800000 diff --git a/src_traditional_2pl_2pc_4_partitions/proto/.DS_Store b/src_traditional_2pl_2pc_4_partitions/proto/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/proto/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/proto/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/proto/Makefile.inc deleted file mode 100755 index 2ad14acc..00000000 --- a/src_traditional_2pl_2pc_4_partitions/proto/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -OBJDIRS += proto - -# Create template specific variables -UPPERC_DIR := PROTO -LOWERC_DIR := proto - -PROTO_SRCS := proto/*.proto - -SRC_LINKED_OBJECTS := $(OBJDIR)/proto/%.cc -TEST_LINKED_OBJECTS := - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Wierd protobuf stuff for compilation -$(OBJDIR)/proto/%.o: $(OBJDIR)/proto/%.pb.h -$(OBJDIR)/proto/%.pb.cc $(OBJDIR)/proto/%.pb.h: proto/%.proto - @echo + protoc $< - @mkdir -p $(@D) - $(V)$(LDLIBRARYPATH) $(PROTOCEXE) --proto_path=proto --cpp_out=$(OBJDIR)/proto $< diff --git a/src_traditional_2pl_2pc_4_partitions/proto/message.proto b/src_traditional_2pl_2pc_4_partitions/proto/message.proto deleted file mode 100755 index 91ddc8d9..00000000 --- a/src_traditional_2pl_2pc_4_partitions/proto/message.proto +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// Protocol buffer used for all network messages in the system. - -message MessageProto { - // Node to which this message should be sent. - required int32 destination_node = 1; - - // Channel to which this message shall be delivered when it arrives at node - // 'destination_node'. - required string destination_channel = 2; - - // Node from which the message originated. - optional int32 source_node = 3; - - // Channel from which the message originated. - optional int32 source_channel = 4; - - // Every type of network message should get an entry here. - enum MessageType { - EMPTY = 0; - TXN_PROTO = 1; - TXN_BATCH = 2; - READ_RESULT = 3; - LINK_CHANNEL = 4; // [Connection implementation specific.] - UNLINK_CHANNEL = 5; // [Connection implementation specific.] - TXN_PTR = 6; - MESSAGE_PTR = 7; - PREPARED = 8; - PREPARED_REPLY = 9; - COMMIT = 10; - COMMIT_REPLY = 11; - TXN_RESTART = 12; - TXN_ABORT = 13; - WAIT_FOR_GRAPH = 14; - }; - required MessageType type = 9; - - // Actual data for the message being carried, to be deserialized into a - // protocol message object of type depending on 'type'. In TXN_PROTO and - // TXN_BATCH messages, 'data' contains are one and any number of TxnProtos, - // respectively. - repeated bytes data = 11; - - // Pointer to actual data for message being carried. Can only be used for - // messages between threads. - repeated int64 data_ptr = 12; - - // For TXN_BATCH messages, 'batch_number' identifies the epoch of the txn - // batch being sent. - optional int64 batch_number = 21; - - // For READ_RESULT messages, 'keys(i)' and 'values(i)' store the key and - // result of a read, respectively. - repeated bytes keys = 31; - repeated bytes values = 32; - - // For (UN)LINK_CHANNEL messages, specifies the main channel of the requesting - // Connection object. - optional string main_channel = 1001; - - // For (UN)LINK_CHANNEL messages, specifies the channel to be (un)linked - // to the requesting Connection object. - optional string channel_request = 1002; - - optional int32 lock_or_release = 1003; - - - repeated int64 wait_txns = 2001; -} - diff --git a/src_traditional_2pl_2pc_4_partitions/proto/tpcc.proto b/src_traditional_2pl_2pc_4_partitions/proto/tpcc.proto deleted file mode 100755 index bc3143fb..00000000 --- a/src_traditional_2pl_2pc_4_partitions/proto/tpcc.proto +++ /dev/null @@ -1,177 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The following are abstract representations of the record types available in -// TPC-C, represented as protocol buffers - -message Warehouse { - // A warehouse has only one primary key - required bytes id = 1; - - // The following are informational fields as defined in TPC-C standard - optional bytes name = 11; - optional bytes street_1 = 12; - optional bytes street_2 = 13; - optional bytes city = 14; - optional bytes state = 15; - optional bytes zip = 16; - - // The following are income records as specified in TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; -} - -message District { - // A district has one primary key and one parent (foreign) key for the - // warehouse it belongs to - required bytes id = 1; - required bytes warehouse_id = 2; - - // The following are informational fields for a district as defined by the - // TPC-C standards - optional bytes name = 10; - optional bytes street_1 = 11; - optional bytes street_2 = 12; - optional bytes city = 13; - optional bytes state = 14; - optional bytes zip = 15; - - // The following are income records as specified in the TPC-C standard - optional double tax = 20; - optional double year_to_date = 21; - optional int32 next_order_id = 22; -} - -message Customer { - // A customer has one primary key, one parent (foreign) key for the district - // it belongs to and one grandparent (foreign) key for the warehouse the - // district it is in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a customer as defined by the - // TPC-C standards - optional bytes first = 10; - optional bytes middle = 11; - required bytes last = 12; - optional bytes street_1 = 13; - optional bytes street_2 = 14; - optional bytes city = 15; - optional bytes state = 16; - optional bytes zip = 17; - - // The following are income records as specified in the TPC-C standard - optional int32 since = 20; - optional bytes credit = 21; - optional double credit_limit = 22; - optional double discount = 23; - optional double balance = 24; - optional double year_to_date_payment = 25; - optional int32 payment_count = 26; - optional int32 delivery_count = 27; - - // The following is an optional data field for entering miscellany - optional bytes data = 30; -} - -message NewOrder { - // A new order has one primary key, one parent (foreign) key for the district - // it originated in, and one grandparent (foreign) key for the warehouse - // the district it originated in belongs to - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; -} - -message Order { - // An order has one primary key, one parent (foreign) key for the customer - // that originated the order, one grandparent (foreign) key for the district - // that customer is in, and one grandparent (foreign) key for the district's - // warehouse - required bytes id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_id = 4; - - // The following are informational fields for an order as defined by the - // TPC-C standards - optional double entry_date = 10; - optional int32 carrier_id = 11; - optional int32 order_line_count = 12; - optional bool all_items_local = 13; - - // Embedding OrderLines into Order (by reference). Kind of hackish. (Alex) - repeated uint64 order_line_ptr = 14; -} - -message OrderLine { - // An order line has a foreign key for the order it belongs to, the district - // the order line occurs in, the warehouse that district belongs to, - // which item is being ordered and which supply warehouse it is being - // taken from - required bytes order_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes item_id = 4; - required bytes supply_warehouse_id = 5; - - // The following are informational fields for an orderline as defined by the - // TPC-C standards - optional int32 number = 10; - optional double delivery_date = 11; - optional int32 quantity = 12; - optional double amount = 13; - optional bytes district_information = 14; -} - -message Item { - // An item has only one primary key - required bytes id = 1; - - // The following are informational fields for an item as defined by the - // TPC-C standards - optional bytes name = 10; - optional double price = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message Stock { - // A stock has one primary key (the item it represents) and one - // foreign key (the warehouse it is in) - required bytes id = 1; - required bytes item_id = 2; - required bytes warehouse_id = 3; - - // The following are informational fields for a stock as defined by the - // TPC-C standards - optional int32 quantity = 10; - repeated bytes districts = 11; - optional int32 year_to_date = 12; - optional int32 order_count = 13; - optional int32 remote_count = 14; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} - -message History { - // A history object contains keys for the customer that originated the - // item, which district and warehouse it was in, and which district and - // warehouse the customer belonged to - required bytes customer_id = 1; - required bytes district_id = 2; - required bytes warehouse_id = 3; - required bytes customer_district_id = 4; - required bytes customer_warehouse_id = 5; - - // The following are informational fields for a history as defined by the - // TPC-C standards - optional double date = 10; - optional double amount = 11; - - // The following is an optional data field for entering miscellany - optional bytes data = 20; -} diff --git a/src_traditional_2pl_2pc_4_partitions/proto/tpcc_args.proto b/src_traditional_2pl_2pc_4_partitions/proto/tpcc_args.proto deleted file mode 100755 index 417fa9ce..00000000 --- a/src_traditional_2pl_2pc_4_partitions/proto/tpcc_args.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// This is a TPC-C specific serializable argset - -message TPCCArgs { - // This represents the system time for the transaction - optional double system_time = 1; - - // Number of warehouses in an initialize txn or load generation - optional bool multipartition = 10; - - // This represents the total order line count for a new order and an array - // of the quantity of item per supply warehouse - optional int32 order_line_count = 20; - repeated int32 quantities = 21; - - // In a payment transaction, this represents the amount of payment - optional int32 amount = 31; - - // Also, we need to record the last name and previous last name (which if - // it does not match means the last name was recently looked up) - optional bytes last_name = 32; -} diff --git a/src_traditional_2pl_2pc_4_partitions/proto/txn.proto b/src_traditional_2pl_2pc_4_partitions/proto/txn.proto deleted file mode 100755 index c8613b12..00000000 --- a/src_traditional_2pl_2pc_4_partitions/proto/txn.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// TODO(alex): Fix types for read_set and write_set. - -message TxnProto { - // Globally unique transaction id, specifying global order. - required int64 txn_id = 1; - - // Specifies which stored procedure to invoke at execution time. - optional int32 txn_type = 10; - - // Isolation level at which to execute transaction. - // - // Note: Currently only full serializability is supported. - enum IsolationLevel { - SERIALIZABLE = 0; - SNAPSHOT = 1; - READ_COMMITTED = 2; - READ_UNCOMMITTED = 3; - }; - optional IsolationLevel isolation_level = 11; - - // True if transaction is known to span multiple database nodes. - optional bool multipartition = 12; - - // Keys of objects read (but not modified) by this transaction. - repeated bytes read_set = 20; - - // Keys of objects modified (but not read) by this transaction. - repeated bytes write_set = 21; - - // Keys of objects read AND modified by this transaction. - repeated bytes read_write_set = 22; - - // Arguments to be passed when invoking the stored procedure to execute this - // transaction. 'arg' is a serialized protocol message. The client and backend - // application code is assumed to know how to interpret this protocol message - // based on 'txn_type'. - optional bytes arg = 23; - - // Transaction status. - // - // TODO(alex): Should this be here? - enum Status { - NEW = 0; - ACTIVE = 1; - COMMITTED = 2; - ABORTED = 3; - }; - optional Status status = 30; - - // Node ids of nodes that participate as readers and writers in this txn. - repeated int32 readers = 40; - repeated int32 writers = 41; - - optional int32 txn_node = 50; - repeated int32 txn_other_node = 51; - -} - diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/.DS_Store b/src_traditional_2pl_2pc_4_partitions/scheduler/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/scheduler/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/scheduler/Makefile.inc deleted file mode 100755 index 381173e0..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/Makefile.inc +++ /dev/null @@ -1,23 +0,0 @@ -OBJDIRS += scheduler - -# Create template specific variables -UPPERC_DIR := SCHEDULER -LOWERC_DIR := scheduler - -SCHEDULER_PROG := -SCHEDULER_SRCS := scheduler/deterministic_lock_manager.cc \ - scheduler/deterministic_scheduler.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) $(BACKEND_OBJS) \ - $(APPLICATION_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-scheduler: $(SCHEDULER_TESTS) - @for a in $(SCHEDULER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.cc b/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.cc deleted file mode 100755 index a745de62..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.cc +++ /dev/null @@ -1,601 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#include -#include - -#include "scheduler/deterministic_lock_manager.h" -#include "sequencer/sequencer.h" - -#include - -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::vector; - -typedef DeterministicLockManager::Latch Latch; - -DeterministicLockManager::DeterministicLockManager(AtomicQueue* ready_txns, Configuration* config) - : configuration_(config), ready_txns_(ready_txns) { - for (int i = 0; i < TABLE_SIZE; i++) - lock_table_[i] = new deque(); - latches_ = new Latch[TABLE_SIZE]; - cond = new pthread_cond_t[WorkersNumber]; - mutex = new pthread_mutex_t[WorkersNumber]; - for(int i = 0; i < WorkersNumber; i++) { - pthread_cond_init(&cond[i], NULL); - pthread_mutex_init(&mutex[i], NULL); - } - pthread_mutex_init(&new_mutex_, NULL); - pthread_mutex_init(&distributed_deadlock_mutex_, NULL); -} - -Latch* DeterministicLockManager::LatchFor(const int& index) { - return latches_ + index; -} - -int DeterministicLockManager::Lock(TxnProto* txn, Connection* connection) { - int not_acquired = 0; - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - int hash_index = Hash(txn->read_write_set(i)); - // Latch acquisitions - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_write_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - // Check whether there is a local deadlock - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - int hash_index = Hash(txn->read_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Record and return the number of locks that the txn is blocked on. - return not_acquired; -} - -int DeterministicLockManager::ContinueLock(TxnProto* txn, Connection* connection) { - int not_acquired = 0; - bool find_point = false; - pthread_mutex_lock(&new_mutex_); - Key point_key = txn_waits_[txn]; - pthread_mutex_unlock(&new_mutex_); - - // Handle read/write lock requests. - for (int i = 0; i < txn->read_write_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_write_set(i))) { - if (find_point == false) { - if (txn->read_write_set(i) == point_key) { - find_point = true; - continue; - } else { - continue; - } - } - - int hash_index = Hash(txn->read_write_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_write_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_write_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(WRITE, txn)); - // Write lock request fails if there is any previous request at all. - if (requests->size() > 1) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_write_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - // Handle read lock requests. This is last so that we don't have to deal with - // upgrading lock requests from read to write on hash collisions. - for (int i = 0; i < txn->read_set_size(); i++) { - // Only lock local keys. - if (IsLocal(txn->read_set(i))) { - if (find_point == false) { - if (txn->read_set(i) == point_key) { - find_point = true; - continue; - } else { - continue; - } - } - - int hash_index = Hash(txn->read_set(i)); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it; - for(it = key_requests->begin(); - it != key_requests->end() && it->key != txn->read_set(i); ++it) { - } - deque* requests; - if (it == key_requests->end()) { - requests = new deque(); - key_requests->push_back(KeysList(txn->read_set(i), requests)); - } else { - requests = it->locksrequest; - } - - // Only need to request this if lock txn hasn't already requested it. - if (requests->empty() || txn != requests->back().txn) { - requests->push_back(LockRequest(READ, txn)); - // Read lock request fails if there is any previous write request. - for (deque::iterator it = requests->begin(); - it != requests->end(); ++it) { - if (it->mode == WRITE) { - not_acquired++; - pthread_mutex_lock(&new_mutex_); - txn_waits_[txn] = txn->read_set(i); - deque::iterator it = requests->begin() + requests->size() - 2; - wait_for_graph[txn->txn_id()] = it->txn->txn_id(); - - bool deadlock = CheckDeadlock(txn); - if (deadlock == true) { - not_acquired = -1; - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); -std::cout<<"~~~~~~ Find a local deadlock, txn id is "<txn_id()<<" related to "<txn->txn_id()<<" \n"<this_node_id); - message.add_wait_txns(it->txn->txn_id()); - bytes txn_data; - txn->SerializeToString(&txn_data); - message.add_data(txn_data); - message.set_lock_or_release(0); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - } - - pthread_mutex_unlock(&new_mutex_); - pthread_mutex_unlock(&latch->lock_); - return not_acquired; - } - } - } - pthread_mutex_unlock(&latch->lock_); - } - } - - if (not_acquired == 0) { - pthread_mutex_lock(&new_mutex_); - txn_waits_.erase(txn); - pthread_mutex_unlock(&new_mutex_); - } - // Record and return the number of locks that the txn is blocked on. - return not_acquired; -} - -void DeterministicLockManager::Release(TxnProto* txn, Connection* connection) { - for (int i = 0; i < txn->read_set_size(); i++) - if (IsLocal(txn->read_set(i))) - Release(txn->read_set(i), txn, connection); - - for (int i = 0; i < txn->read_write_set_size(); i++) - if (IsLocal(txn->read_write_set(i))) - Release(txn->read_write_set(i), txn, connection); -} - -void DeterministicLockManager::Release(const Key& key, TxnProto* txn, Connection* connection) { - int hash_index = Hash(key); - Latch* latch = LatchFor(hash_index); - pthread_mutex_lock(&latch->lock_); - // Avoid repeatedly looking up key in the unordered_map. - deque* key_requests = lock_table_[hash_index]; - - deque::iterator it1; - for(it1 = key_requests->begin(); - it1 != key_requests->end() && it1->key != key; ++it1) { - } - - if (it1 == key_requests->end()) { - pthread_mutex_unlock(&latch->lock_); - return ; - } - - deque* requests = it1->locksrequest; - - // Seek to the target request. Note whether any write lock requests precede - // the target. - bool write_requests_precede_target = false; - deque::iterator it; - for (it = requests->begin(); - it != requests->end() && it->txn != txn; ++it) { - if (it->mode == WRITE) - write_requests_precede_target = true; - } - - // If we found the request, erase it. No need to do anything otherwise. - if (it != requests->end()) { - // Save an iterator pointing to the target to call erase on after handling - // lock inheritence, since erase(...) trashes all iterators. - deque::iterator target = it; - - // If there are more requests following the target request, one or more - // may need to be granted as a result of the target's release. - ++it; - if (it != requests->end()) { - vector new_owners; - // Grant subsequent request(s) if: - // (a) The canceled request held a write lock. - // (b) The canceled request held a read lock ALONE. - // (c) The canceled request was a write request preceded only by read - // requests and followed by one or more read requests. - if (target == requests->begin() && - (target->mode == WRITE || - (target->mode == READ && it->mode == WRITE))) { // (a) or (b) - // If a write lock request follows, grant it. - if (it->mode == WRITE) - new_owners.push_back(it->txn); - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } else if (!write_requests_precede_target && - target->mode == WRITE && it->mode == READ) { // (c) - // If a sequence of read lock requests follows, grant all of them. - for (; it != requests->end() && it->mode == READ; ++it) - new_owners.push_back(it->txn); - } - - // Handle txns with newly granted requests that may now be ready to run. - pthread_mutex_lock(&new_mutex_); - for (uint64 j = 0; j < new_owners.size(); j++) { - // The txn that just acquired the released lock is no longer waiting - // on any lock requests. - if (doing_deadlocks.count(IntToString(new_owners[j]->txn_id())) == 0) { - ready_txns_->Push(new_owners[j]); - } - - if (wait_for_graph.count(new_owners[j]->txn_id()) > 0) - wait_for_graph.erase(new_owners[j]->txn_id()); - - // Send message to machine 0, to delete blocked information from global wait for graph. - MessageProto message; - message.set_type(MessageProto::WAIT_FOR_GRAPH); - message.set_destination_channel("wait_for_graph"); - message.set_destination_node(0); - message.set_source_node(configuration_->this_node_id); - message.add_wait_txns(new_owners[j]->txn_id()); - message.add_wait_txns(txn->txn_id()); - message.set_lock_or_release(1); - if (configuration_->this_node_id == 0) - connection->Send(message); - else - connection->Send1(message); - - } - pthread_mutex_unlock(&new_mutex_); - - } - - // Now it is safe to actually erase the target request. - requests->erase(target); - if (requests->size() == 0) { - delete requests; - key_requests->erase(it1); - } - } - pthread_mutex_unlock(&latch->lock_); -} - -// Local deadlock detection -bool DeterministicLockManager::CheckDeadlock(TxnProto* txn) { - bool deadlock = false; - int64 next_txn = wait_for_graph[txn->txn_id()]; - while (true) { - if (wait_for_graph.count(next_txn) == 0) { - deadlock = false; - break; - } else if (wait_for_graph[next_txn] == txn->txn_id()) { - deadlock = true; - break; - } - next_txn = wait_for_graph[next_txn]; - } - - return deadlock; -} - -// Distributed deadlock detection -bool DeterministicLockManager::CheckDistributedDeadlock(int64 txn1_id, int64 txn2_id) { - bool deadlock = false; - - vector aa = distributed_wait_for_graph[txn2_id]; -set tt; - for (int i = 0; i < (int)aa.size(); i ++) { - int64 bb = aa[i]; - if (bb == txn1_id) { - deadlock = true; - return deadlock; - } - - if (distributed_wait_for_graph.count(bb) > 0 && tt.count(bb) == 0) { - deadlock = CheckDistributedDeadlock(txn1_id, bb); - if (deadlock == true) { - return deadlock; - } - tt.insert(bb); - } - } - - return deadlock; -} - -void DeterministicLockManager::HandleDistributedDeadlock(MessageProto message, Connection* connection) { - - int lock_or_release = message.lock_or_release(); - - if (lock_or_release == 0) { - TxnProto* txn1 = new TxnProto(); - txn1->ParseFromString(message.data(0)); - int64 txn1_id = txn1->txn_id(); - int64 txn2_id = message.wait_txns(0); - - if (distributed_wait_for_graph.count(txn1_id) > 0) { - vector aa = distributed_wait_for_graph[txn1_id]; - for(int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].push_back(txn2_id); - return; - } - } - } - - distributed_wait_for_graph[txn1_id].push_back(txn2_id); - bool deadlock = CheckDistributedDeadlock(txn1_id, txn2_id); - if (deadlock == true) { - - vector aa = distributed_wait_for_graph[txn1_id]; - for (int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].erase(distributed_wait_for_graph[txn1_id].begin() + i); - } - } - if (distributed_wait_for_graph[txn1_id].size() == 0) { - distributed_wait_for_graph.erase(txn1_id); - } - -std::cout<<"!!!!!!: find a distributed deadlock, txn is "<txn_node()); - connection->Send(message1); - - for (int i = 0; i < txn1->txn_other_node_size(); i++) { - MessageProto message2; - message2.set_type(MessageProto::TXN_ABORT); - message2.set_destination_channel(IntToString(txn1_id)); - message2.set_destination_node(txn1->txn_other_node(i)); - connection->Send(message2); - } - - } - } else { - int64 txn1_id = message.wait_txns(0); - int64 txn2_id = message.wait_txns(1); - if (distributed_wait_for_graph.count(txn1_id) > 0) { - vector aa = distributed_wait_for_graph[txn1_id]; - for (int i = 0; i < (int)aa.size(); i++) { - if (aa[i] == txn2_id) { - distributed_wait_for_graph[txn1_id].erase(distributed_wait_for_graph[txn1_id].begin() + i); - break; - } - } - if (distributed_wait_for_graph[txn1_id].size() == 0) { - distributed_wait_for_graph.erase(txn1_id); - } - } - } -} - -int DeterministicLockManager::try_deadlock_mutex() { - int ret = pthread_mutex_trylock(&distributed_deadlock_mutex_); - return ret; -} - -void DeterministicLockManager::release_deadlock_mutex() { - pthread_mutex_unlock(&distributed_deadlock_mutex_); -} - -TxnProto* DeterministicLockManager::GetBlockedTxn(string txn_id) { - pthread_mutex_lock(&new_mutex_); - TxnProto* txn = NULL; - for (unordered_map::iterator it = txn_waits_.begin(); - it != txn_waits_.end(); ++it) { - if (it->first->txn_id() == StringToInt(txn_id)) { - txn = it->first; - break; - } - } - - if (txn != NULL) { - txn_waits_.erase(txn); - wait_for_graph.erase(txn->txn_id()); - } - - pthread_mutex_unlock(&new_mutex_); - return txn; -} - - - diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.h b/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.h deleted file mode 100755 index afa4395b..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_lock_manager.h +++ /dev/null @@ -1,114 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// Lock manager implementing deterministic two-phase locking as described in -// 'The Case for Determinism in Database Systems'. - -#ifndef _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ - -#include -#include -#include - -#include "common/configuration.h" -#include "scheduler/lock_manager.h" -#include "common/utils.h" -#include "common/connection.h" - -using std::tr1::unordered_map; -using std::deque; -using std::pair; -using std::vector; - -#define TABLE_SIZE 1000000 - -class TxnProto; - -class DeterministicLockManager { - public: - DeterministicLockManager(AtomicQueue* ready_txns, Configuration* config); - virtual ~DeterministicLockManager() {} - virtual int Lock(TxnProto* txn, Connection* connection); - virtual int ContinueLock(TxnProto* txn, Connection* connection); - virtual void Release(TxnProto* txn, Connection* connection); - virtual void Release(const Key& key, TxnProto* txn, Connection* connection); - virtual bool CheckDeadlock(TxnProto* txn); - virtual bool CheckDistributedDeadlock(int64 txn1_id, int64 txn2_id); - virtual void HandleDistributedDeadlock(MessageProto message, Connection* connection); - virtual int try_deadlock_mutex(); - virtual void release_deadlock_mutex(); - virtual TxnProto* GetBlockedTxn(string txn_id); - - class Latch { - public: - pthread_mutex_t lock_; - - Latch() { - pthread_mutex_init(&lock_, NULL); - } - }; - Latch* LatchFor(const int& index); - -pthread_cond_t* cond; -pthread_mutex_t* mutex; -unordered_map txn_waits_; - - private: - int Hash(const Key& key) { - uint64 hash = 2166136261; - for (size_t i = 0; i < key.size(); i++) { - hash = hash ^ (key[i]); - hash = hash * 16777619; - } - return hash % TABLE_SIZE; - //return atoi(key.c_str()); - } - - bool IsLocal(const Key& key) { - return configuration_->LookupPartition(key) == configuration_->this_node_id; - } - - // Configuration object (needed to avoid locking non-local keys). - Configuration* configuration_; - - // The DeterministicLockManager's lock table tracks all lock requests. For a - // given key, if 'lock_table_' contains a nonempty queue, then the item with - // that key is locked and either: - // (a) first element in the queue specifies the owner if that item is a - // request for a write lock, or - // (b) a read lock is held by all elements of the longest prefix of the queue - // containing only read lock requests. - // Note: using STL deque rather than queue for erase(iterator position). - struct LockRequest { - LockRequest(LockMode m, TxnProto* t) : txn(t), mode(m) {} - TxnProto* txn; // Pointer to txn requesting the lock. - LockMode mode; // Specifies whether this is a read or write lock request. - }; - - struct KeysList { - KeysList(Key m, deque* t) : key(m), locksrequest(t) {} - Key key; - deque* locksrequest; - }; - - - - deque* lock_table_[TABLE_SIZE]; - - Latch* latches_; - - AtomicQueue* ready_txns_; - - // Tracks all txns still waiting on acquiring at least one lock. Entries in - // 'txn_waits_' are invalided by any call to Release() with the entry's - // txn. - - pthread_mutex_t new_mutex_; - - pthread_mutex_t distributed_deadlock_mutex_; - - unordered_map wait_for_graph; - - unordered_map> distributed_wait_for_graph; -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.cc b/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.cc deleted file mode 100755 index c4bd6435..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.cc +++ /dev/null @@ -1,505 +0,0 @@ -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/deterministic_scheduler.h" - -#include -#include -#include -#include -#include -#include - -#include "applications/application.h" -#include "common/utils.h" -#include "common/connection.h" -#include "backend/storage.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#include "scheduler/deterministic_lock_manager.h" - -// XXX(scw): why the F do we include from a separate component -// to get COLD_CUTOFF -#include "sequencer/sequencer.h" // COLD_CUTOFF and buffers in LATENCY_TEST - -using std::pair; -using std::string; -using std::tr1::unordered_map; -using std::bitset; - - -DeterministicScheduler::DeterministicScheduler(Configuration* conf, - ConnectionMultiplexer* multiplexer, - Storage* storage, - const Application* application) - : configuration_(conf), multiplexer_(multiplexer), storage_(storage){ - application_ = application; -pthread_mutex_init(&test_mutex_, NULL); -pthread_mutex_init(&batch_mutex_, NULL); - for (int i = 0; i < WorkersNumber; i++) { - message_queues[i] = new AtomicQueue(); - } - - batch_txns = new AtomicQueue(); - batch_id = 0; - - string channel_batch("scheduler_"); - batch_connections_ = multiplexer_->NewConnection(channel_batch); - -cpu_set_t cpuset; - - for (int i = 0; i < WorkersNumber; i++) { - string channel_worker("worker"); - channel_worker.append(IntToString(i)); - thread_connections_[i] = multiplexer_->NewConnection(channel_worker, &message_queues[i]); -CPU_ZERO(&cpuset); -pthread_attr_t attr; -pthread_attr_init(&attr); -if (i == 0 || i == 1) -CPU_SET(i, &cpuset); -else if (i == 2 || i == 3) -CPU_SET(i+2, &cpuset); -else -CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); - pthread_create(&(threads_[i]), &attr, RunWorkerThread, reinterpret_cast( - new pair(configuration_->this_node_id * WorkersNumber + i, this))); - } - - ready_txns_ = new AtomicQueue(); - lock_manager_ = new DeterministicLockManager(ready_txns_, configuration_); - - pthread_mutex_init(&deadlock_mutex_, NULL); - - if (configuration_->this_node_id == 0) { - string channel_worker("wait_for_graph"); - distributed_deadlock_queue = new AtomicQueue(); - distributed_deadlock_ = multiplexer_->NewConnection(channel_worker, &distributed_deadlock_queue); - } - - string channel_worker("force_abort_unactive_txn"); - abort_txn_queue = new AtomicQueue(); - abort_txn_ = multiplexer_->NewConnection(channel_worker, &abort_txn_queue); -} - -DeterministicScheduler::~DeterministicScheduler() { -} - -// Returns ptr to heap-allocated -unordered_map batches; -int last_txn = -1;; -void GetBatch(DeterministicScheduler* scheduler) { - if(pthread_mutex_trylock(&scheduler->batch_mutex_) == 0) { - MessageProto* message = new MessageProto(); - if (scheduler->batch_connections_->GetMessage(message)) { - assert(message->type() == MessageProto::TXN_BATCH); - batches[message->batch_number()] = message; - if (batches.count(scheduler->batch_id) > 0) { - message = batches[scheduler->batch_id]; - for (int i = 0; i < message->data_size(); i++) { - TxnProto* txn = new TxnProto(); - txn->ParseFromString(message->data(i)); - scheduler->batch_txns->Push(txn); - last_txn = txn->txn_id(); - } - batches.erase(scheduler->batch_id); - scheduler->batch_id ++; - delete message; - } - } else { - delete message; - } - - pthread_mutex_unlock(&scheduler->batch_mutex_); - } -} - -void* DeterministicScheduler::RunWorkerThread(void* arg) { - - int partition_id = - reinterpret_cast*>(arg)->first; - DeterministicScheduler* scheduler = - reinterpret_cast*>(arg)->second; - - int thread = partition_id % WorkersNumber; - MessageProto message; - - int txns = 0; - double start_time = GetTime(); - double time = GetTime(); - - Storage* this_storage = scheduler->storage_; - int this_node_id = scheduler->configuration_->this_node_id; - - unordered_map active_txns; - - Spin(2); - - while(true) { - bool got_message = scheduler->abort_txn_queue->Pop(&message); - if (got_message == true) { - TxnProto* txn = scheduler->lock_manager_->GetBlockedTxn(message.destination_channel()); - if (txn == NULL) { - continue; - } - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - doing_deadlocks.erase(IntToString(txn->txn_id())); - if (txn->txn_node() == this_node_id) { - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->thread_connections_[thread]->Send(restart); - } - delete txn; - } - - got_message = scheduler->message_queues[thread]->Pop(&message); - if (got_message == true) { - switch (message.type()) { - case MessageProto::TXN_ABORT: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - TxnProto* txn = manager->txn_; - assert(txn != NULL); - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->txn_node() == this_node_id) { - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } - doing_deadlocks.erase(IntToString(txn->txn_id())); - break; - } - - case MessageProto::READ_RESULT: - { - if (active_txns.count(message.destination_channel()) == 0) { - break; - } - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandleReadResult(message); - TxnProto* txn = manager->txn_; - if (manager->ReadyToExecute()) { - scheduler->application_->Execute(txn, manager); - // If this node is the coordinator, broadcast prepared messages - if (txn->txn_node() == this_node_id) { - manager->BroadcastPreparedMessages(); - } - if (manager->ReceivedPreparedMessages() == true) { - manager->SendPreparedReplyMessages(); - } - } - break; - } - - case MessageProto::PREPARED: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandlePreparedMessages(message); - if (manager->ReadyToExecute()) { - manager->SendPreparedReplyMessages(); - } - break; - } - - case MessageProto::PREPARED_REPLY: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - manager->HandlePreparedReplyMessages(message); - if (manager->ReceivedAllPreparedReplyMessages()) { - manager->BroadcastCommitMessages(); - } - break; - } - - case MessageProto::COMMIT: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - TxnProto* txn = manager->txn_; - assert(txn != NULL); - manager->HandleCommitMessages(message); - manager->SendCommitReplyMessages(); - // Done the transaction - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if((txn->writers_size() == 0) && (rand() % txn->readers_size() == 0)) { - txns++; - } else if(rand() % txn->writers_size() == 0) { - txns++; - } - delete txn; - delete manager; - break; - } - - case MessageProto::COMMIT_REPLY: - { - assert(active_txns.count(message.destination_channel())>0); - StorageManager* manager = active_txns[message.destination_channel()]; - assert(manager != NULL); - TxnProto* txn = manager->txn_; - manager->HandleCommitReplyMessages(message); - if (manager->ReceivedAllCommitReplyMessages()) { - // Done the transaction - active_txns.erase(message.destination_channel()); - scheduler->thread_connections_[thread]->UnlinkChannel(message.destination_channel()); - - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - - if((txn->writers_size() == 0) && (rand() % txn->readers_size() == 0)) { - txns++; - } else if(rand() % txn->writers_size() == 0) { - txns++; - } - delete txn; - delete manager; - } - break; - } - - default: - break; - } - - } - -if ((int)active_txns.size() < 100 && scheduler->lock_manager_->txn_waits_.size() < 40) { - TxnProto* txn; - bool got_message = scheduler->batch_txns->Pop(&txn); - if (got_message == true) { - if (doing_deadlocks.count(IntToString(txn->txn_id())) > 0) { - doing_deadlocks.erase(IntToString(txn->txn_id())); - continue; - } - - int ret = scheduler->lock_manager_->Lock(txn, scheduler->thread_connections_[thread]); - // Have acquired all locks, execute the txn now. - if(ret == 0) { - - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - this_storage, txn, this_node_id); - - if (manager->ReadyToExecute()) { - // Execute the txn. - scheduler->application_->Execute(txn, manager); - - // If don't happen deadlocks, just release all locks - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - delete manager; - delete txn; - txns++; - } else { - string channel = IntToString(txn->txn_id()); - scheduler->thread_connections_[thread]->LinkChannel(channel); - active_txns[channel] = manager; - } - - } else if (ret == -1) { - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->multipartition() == false) { - // Single-partition txn - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else if (txn->multipartition() == true) { - // Multi-partition txn - if (txn->txn_node() == this_node_id) { - for (int i = 0; i < txn->txn_other_node_size(); i++) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - - scheduler->thread_connections_[thread]->Send(restart); - } else { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_node()); - scheduler->thread_connections_[thread]->Send1(message); - - for (int i = 0; i < txn->txn_other_node_size(); i++) { - if (txn->txn_other_node(i) != this_node_id) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - } - } - - } - } - } -} - - - if (!scheduler->ready_txns_->Empty()) { - TxnProto* txn; - bool got_it = scheduler->ready_txns_->Pop(&txn); - if (got_it == true) { - if (doing_deadlocks.count(IntToString(txn->txn_id())) > 0) { - doing_deadlocks.erase(IntToString(txn->txn_id())); - continue; - } - int ret = scheduler->lock_manager_->ContinueLock(txn, scheduler->thread_connections_[thread]); - if (ret == 0) { - StorageManager* manager = new StorageManager(scheduler->configuration_, - scheduler->thread_connections_[thread], - this_storage, txn, this_node_id); - if (manager->ReadyToExecute()) { - // Execute the txn. - scheduler->application_->Execute(txn, manager); - - // If don't happen deadlocks, just release all locks - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - - delete manager; - delete txn; - txns++; - } else { - string channel = IntToString(txn->txn_id()); - scheduler->thread_connections_[thread]->LinkChannel(channel); - active_txns[channel] = manager; - } - } else if (ret == -1) { - scheduler->lock_manager_->Release(txn, scheduler->thread_connections_[thread]); - if (txn->multipartition() == false) { - // Single-partition txn - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else if (txn->multipartition() == true) { - // Multi-partition txn - if (txn->txn_node() == this_node_id) { - for (int i = 0; i < txn->txn_other_node_size(); i++) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - - MessageProto restart; - restart.set_destination_channel("sequencer"); - restart.set_destination_node(this_node_id); - restart.set_type(MessageProto::TXN_RESTART); - bytes txn_data; - txn->clear_readers(); - txn->clear_writers(); - txn->SerializeToString(&txn_data); - restart.add_data(txn_data); - scheduler->thread_connections_[thread]->Send(restart); - } else { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_node()); - scheduler->thread_connections_[thread]->Send1(message); - - for (int i = 0; i < txn->txn_other_node_size(); i++) { - if (txn->txn_other_node(i) != this_node_id) { - MessageProto message; - message.set_type(MessageProto::TXN_ABORT); - message.set_destination_channel(IntToString(txn->txn_id())); - message.set_destination_node(txn->txn_other_node(i)); - scheduler->thread_connections_[thread]->Send1(message); - } - } - } - } - } - } - } - - if (this_node_id == 0 && scheduler->lock_manager_->try_deadlock_mutex() == 0) { - got_message = scheduler->distributed_deadlock_queue->Pop(&message); - if (got_message == true) { - scheduler->lock_manager_->HandleDistributedDeadlock(message, scheduler->thread_connections_[thread]); - scheduler->lock_manager_->release_deadlock_mutex(); - } else { - scheduler->lock_manager_->release_deadlock_mutex(); - } - } - - if (txns % 10 == 0) { - GetBatch(scheduler); - } - - if(GetTime() > time + 1) { - double total_time = GetTime() - time; - std::cout<<"Partition "<(txns)/total_time)<<" txns/sec. "<<(int)active_txns.size()<<" "<lock_manager_->txn_waits_.size()<<"\n"< start_time + 240) - exit(0); -} - - } - - diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.h b/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.h deleted file mode 100755 index 5f43b3f7..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/deterministic_scheduler.h +++ /dev/null @@ -1,89 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The deterministic lock manager implements deterministic locking as described -// in 'The Case for Determinism in Database Systems', VLDB 2010. Each -// transaction must request all locks it will ever need before the next -// transaction in the specified order may acquire any locks. Each lock is then -// granted to transactions in the order in which they requested them (i.e. in -// the global transaction order). - -#ifndef _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ -#define _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ - -#include - -#include -#include - -#include "scheduler/scheduler.h" -#include -#include "common/connection.h" -#include "common/configuration.h" -#include "scheduler/deterministic_lock_manager.h" -#include "common/utils.h" -#include "proto/txn.pb.h" -#include "proto/message.pb.h" - -using std::deque; -using std::tr1::unordered_map; -using std::atomic; - -class Configuration; -class Connection; -class ConnectionMultiplexer; -class DeterministicLockManager; -class Storage; -class TxnProto; -class Application; - -class DeterministicScheduler : public Scheduler { - public: - DeterministicScheduler(Configuration* conf, ConnectionMultiplexer* multiplexer, - Storage* storage, const Application* application); - virtual ~DeterministicScheduler(); - - // Connection for receiving txn batches from sequencer. - Connection* batch_connections_; - pthread_mutex_t test_mutex_; - pthread_mutex_t batch_mutex_; - AtomicQueue* batch_txns; - int batch_id; - - private: - // Application currently being run. - const Application* application_; - - // Function for starting main loops in a separate pthreads. - static void* RunWorkerThread(void* arg); - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Thread contexts and their associated Connection objects. - pthread_t threads_[WorkersNumber]; - Connection* thread_connections_[WorkersNumber]; - - ConnectionMultiplexer* multiplexer_; - - // Storage layer used in application execution. - Storage* storage_; - - DeterministicLockManager* lock_manager_; - - AtomicQueue* message_queues[WorkersNumber]; - - AtomicQueue* ready_txns_; - - atomic blocked; - - pthread_mutex_t deadlock_mutex_; - - Connection* distributed_deadlock_; - AtomicQueue* distributed_deadlock_queue; - - Connection* abort_txn_; - AtomicQueue* abort_txn_queue; - -}; -#endif // _DB_SCHEDULER_DETERMINISTIC_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/lock_manager.h b/src_traditional_2pl_2pc_4_partitions/scheduler/lock_manager.h deleted file mode 100755 index 6bc211cc..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/lock_manager.h +++ /dev/null @@ -1,52 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// Interface for lock managers in the system. - -#ifndef _DB_SCHEDULER_LOCK_MANAGER_H_ -#define _DB_SCHEDULER_LOCK_MANAGER_H_ - -#include - -#include "common/types.h" - -using std::vector; - -class TxnProto; - -// This interface supports locks being held in both read/shared and -// write/exclusive modes. -enum LockMode { - UNLOCKED = 0, - READ = 1, - WRITE = 2, -}; - -class LockManager { - public: - virtual ~LockManager() {} - // Attempts to assign the lock for each key in keys to the specified - // transaction. Returns the number of requested locks NOT assigned to the - // transaction (therefore Lock() returns 0 if the transaction successfully - // acquires all locks). - // - // Requires: 'read_keys' and 'write_keys' do not overlap, and neither contains - // duplicate keys. - // Requires: Lock has not previously been called with this txn_id. Note that - // this means Lock can only ever be called once per txn. - virtual int Lock(TxnProto* txn) = 0; - - // For each key in 'keys': - // - If the specified transaction owns the lock on the item, the lock is - // released. - // - If the transaction is in the queue to acquire a lock on the item, the - // request is cancelled and the transaction is removed from the item's - // queue. - virtual void Release(const Key& key, TxnProto* txn) = 0; - virtual void Release(TxnProto* txn) = 0; - - // Locked sets '*owner' to contain the txn IDs of all txns holding the lock, - // and returns the current state of the lock: UNLOCKED if it is not currently - // held, READ or WRITE if it is, depending on the current state. - virtual LockMode Status(const Key& key, vector* owners) = 0; -}; -#endif // _DB_SCHEDULER_LOCK_MANAGER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/scheduler.h b/src_traditional_2pl_2pc_4_partitions/scheduler/scheduler.h deleted file mode 100755 index 45557a1b..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/scheduler.h +++ /dev/null @@ -1,18 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// A database node's Scheduler determines what transactions should be run when -// at that node. It is responsible for communicating with other nodes when -// necessary to determine whether a transaction can be scheduled. It also -// forwards messages on to the backend that are sent from other nodes -// participating in distributed transactions. - -#ifndef _DB_SCHEDULER_SCHEDULER_H_ -#define _DB_SCHEDULER_SCHEDULER_H_ - -class Application; - -class Scheduler { - public: - virtual ~Scheduler() {} -}; -#endif // _DB_SCHEDULER_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.cc b/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.cc deleted file mode 100755 index 1d2c9bfe..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. -// -// TODO(scw): replace iostream with cstdio - -#include "scheduler/serial_scheduler.h" - -#include - -#include "applications/application.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "backend/storage_manager.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" - -SerialScheduler::SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing) - : configuration_(conf), connection_(connection), storage_(storage), - checkpointing_(checkpointing) { -} - -SerialScheduler::~SerialScheduler() {} - -void SerialScheduler::Run(const Application& application) { - MessageProto message; - TxnProto txn; - StorageManager* manager; - Connection* manager_connection = - connection_->multiplexer()->NewConnection("manager_connection"); - - int txns = 0; - double time = GetTime(); - double start_time = time; - while (true) { - if (connection_->GetMessage(&message)) { - // Execute all txns in batch. - for (int i = 0; i < message.data_size(); i++) { - txn.ParseFromString(message.data(i)); - - // Link txn-specific channel ot manager_connection. - manager_connection->LinkChannel(IntToString(txn.txn_id())); - - // Create manager. - manager = new StorageManager(configuration_, manager_connection, - storage_, &txn); - - // Execute txn if any writes occur at this node. - if (manager->writer) { - while (!manager->ReadyToExecute()) { - if (connection_->GetMessage(&message)) - manager->HandleReadResult(message); - } - application.Execute(&txn, manager); - } - // Clean up the mess. - delete manager; - manager_connection->UnlinkChannel(IntToString(txn.txn_id())); - - // Report throughput (once per second). TODO(alex): Fix reporting. - if (txn.writers(txn.txn_id() % txn.writers_size()) == - configuration_->this_node_id) - txns++; - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - } - } - - // Report throughput (once per second). - if (GetTime() > time + 1) { - std::cout << "Executed " << txns << " txns\n" << std::flush; - // Reset txn count. - time = GetTime(); - txns = 0; - } - - // Run for at most one minute. - if (GetTime() > start_time + 60) - exit(0); - } - - delete manager_connection; -} diff --git a/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.h b/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.h deleted file mode 100755 index f99958c6..00000000 --- a/src_traditional_2pl_2pc_4_partitions/scheduler/serial_scheduler.h +++ /dev/null @@ -1,35 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// SerialScheduler is a trivial scheduler that executes transactions serially -// as they come in, without locking. - -#ifndef _DB_SCHEDULER_SERIAL_SCHEDULER_H_ -#define _DB_SCHEDULER_SERIAL_SCHEDULER_H_ - -#include "scheduler/scheduler.h" - -class Configuration; -class Connection; -class Storage; - -class SerialScheduler : public Scheduler { - public: - SerialScheduler(Configuration* conf, Connection* connection, - Storage* storage, bool checkpointing); - virtual ~SerialScheduler(); - virtual void Run(const Application& application); - - private: - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Storage layer used in application execution. - Storage* storage_; - - // Should we checkpoint? - bool checkpointing_; -}; -#endif // _DB_SCHEDULER_SERIAL_SCHEDULER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/sequencer/.DS_Store b/src_traditional_2pl_2pc_4_partitions/sequencer/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/src_traditional_2pl_2pc_4_partitions/sequencer/.DS_Store and /dev/null differ diff --git a/src_traditional_2pl_2pc_4_partitions/sequencer/Makefile.inc b/src_traditional_2pl_2pc_4_partitions/sequencer/Makefile.inc deleted file mode 100755 index 24d71b4e..00000000 --- a/src_traditional_2pl_2pc_4_partitions/sequencer/Makefile.inc +++ /dev/null @@ -1,21 +0,0 @@ -OBJDIRS += sequencer - -# Create template specific variables -UPPERC_DIR := SEQUENCER -LOWERC_DIR := sequencer - -SEQUENCER_PROG := -SEQUENCER_SRCS := sequencer/sequencer.cc - -SRC_LINKED_OBJECTS := -TEST_LINKED_OBJECTS := $(PROTO_OBJS) $(COMMON_OBJS) - -# Link the template to avoid redundancy -include $(MAKEFILE_TEMPLATE) - -# Need to specify test cases explicitly because they have variables in recipe -test-sequencer: $(SEQUENCER_TESTS) - @for a in $(SEQUENCER_TESTS); do \ - echo == $$a ==; \ - $(LDLIBRARYPATH) $$a; \ - done diff --git a/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.cc b/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.cc deleted file mode 100755 index 6dfd55f0..00000000 --- a/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.cc +++ /dev/null @@ -1,294 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. -// -// TODO(scw): replace iostream with cstdio - -#include "sequencer/sequencer.h" - -#include -#include -#include -#include -#include - -#include "backend/storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/utils.h" -#include "proto/message.pb.h" -#include "proto/txn.pb.h" -#ifdef PAXOS -# include "paxos/paxos.h" -#endif - -using std::map; -using std::multimap; -using std::set; -using std::queue; - -#ifdef LATENCY_TEST1 -double execute_begin[SAMPLES]; -double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -double sequencer_recv[SAMPLES]; -// double paxos_begin[SAMPLES]; -// double paxos_end[SAMPLES]; -double sequencer_send[SAMPLES]; -double prefetch_cold[SAMPLES]; -double scheduler_lock[SAMPLES]; -double worker_begin[SAMPLES]; -double worker_end[SAMPLES]; -double scheduler_unlock[SAMPLES]; -#endif - -void* Sequencer::RunSequencerWriter(void *arg) { - reinterpret_cast(arg)->RunWriter(); - return NULL; -} - -void* Sequencer::RunSequencerReader(void *arg) { - reinterpret_cast(arg)->RunReader(); - return NULL; -} - -Sequencer::Sequencer(Configuration* conf, Connection* connection, - Client* client) - : epoch_duration_(0.01), configuration_(conf), connection_(connection), - client_(client), deconstructor_invoked_(false) { - pthread_mutex_init(&mutex_, NULL); - -cpu_set_t cpuset; -pthread_attr_t attr_writer; -pthread_attr_init(&attr_writer); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -CPU_SET(2, &cpuset); -//CPU_SET(7, &cpuset); -pthread_attr_setaffinity_np(&attr_writer, sizeof(cpu_set_t), &cpuset); - - - // Start Sequencer main loops running in background thread. - pthread_create(&writer_thread_, &attr_writer, RunSequencerWriter, - reinterpret_cast(this)); - -CPU_ZERO(&cpuset); -//CPU_SET(4, &cpuset); -//CPU_SET(5, &cpuset); -//CPU_SET(6, &cpuset); -CPU_SET(6, &cpuset); -pthread_attr_t attr_reader; -pthread_attr_init(&attr_reader); -pthread_attr_setaffinity_np(&attr_reader, sizeof(cpu_set_t), &cpuset); - - pthread_create(&reader_thread_, &attr_reader, RunSequencerReader, - reinterpret_cast(this)); -} - -Sequencer::~Sequencer() { -std::cout << "~Sequencer() execute.\n" << std::flush; - deconstructor_invoked_ = true; - pthread_join(writer_thread_, NULL); - pthread_join(reader_thread_, NULL); -} - -void Sequencer::FindParticipatingNodes(const TxnProto& txn, set* nodes) { - nodes->clear(); - for (int i = 0; i < txn.read_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) - nodes->insert(configuration_->LookupPartition(txn.read_write_set(i))); -} - -void Sequencer::RunWriter() { - Spin(1); - -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, false); -#endif - - // Synchronization loadgen start with other sequencers. - MessageProto synchronization_message; - synchronization_message.set_type(MessageProto::EMPTY); - synchronization_message.set_destination_channel("sequencer"); - for (uint32 i = 0; i < configuration_->all_nodes.size(); i++) { - synchronization_message.set_destination_node(i); - if (i != static_cast(configuration_->this_node_id)) - connection_->Send(synchronization_message); - } - uint32 synchronization_counter = 1; - while (synchronization_counter < configuration_->all_nodes.size()) { - synchronization_message.Clear(); - if (connection_->GetMessage(&synchronization_message)) { - assert(synchronization_message.type() == MessageProto::EMPTY); - synchronization_counter++; - } - } -//std::cout << "Starting sequencer, enter RunWriter.\n" << std::flush; - - // Set up batch messages for each system node. - MessageProto batch; - batch.set_destination_channel("sequencer"); - batch.set_destination_node(-1); - string batch_string; - batch.set_type(MessageProto::TXN_BATCH); -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - for (int batch_number = configuration_->this_node_id; - !deconstructor_invoked_; - batch_number += configuration_->all_nodes.size()) { - // Begin epoch. - double epoch_start = GetTime(); - batch.set_batch_number(batch_number); - batch.clear_data(); - - // Collect txn requests for this epoch. - int txn_id_offset = 0; - while (!deconstructor_invoked_ && - GetTime() < epoch_start + epoch_duration_) { - // Add next txn request to batch. - if (batch.data_size() < MAX_BATCH_SIZE) { - TxnProto* txn; - string txn_string; - MessageProto message; - if (connection_->GetMessage(&message)) { - txn = new TxnProto(); - assert(message.type() == MessageProto::TXN_RESTART); - txn->ParseFromString(message.data(0)); - txn->set_txn_id(batch_number * MAX_BATCH_SIZE + txn_id_offset); - txn_id_offset++; -std::cout<<"~~~~ Sequencer received a MessageProto::TXN_RESTART message, txn id is "<txn_id()<<"\n"<GetTxn(&txn, batch_number * MAX_BATCH_SIZE + txn_id_offset); - txn_id_offset++; - } - - txn->SerializeToString(&txn_string); - batch.add_data(txn_string); - delete txn; - } - } - // Send this epoch's requests to Paxos service. - batch.SerializeToString(&batch_string); -#ifdef PAXOS - paxos.SubmitBatch(batch_string); -#else - pthread_mutex_lock(&mutex_); - batch_queue_.push(batch_string); - pthread_mutex_unlock(&mutex_); -#endif - } - - Spin(1); -} - -void Sequencer::RunReader() { - Spin(1); -#ifdef PAXOS - Paxos paxos(ZOOKEEPER_CONF, true); -#endif - // Set up batch messages for each system node. - map batches; - for (int i = 0; i < (int)configuration_->all_nodes.size(); i++) { - string channel("scheduler_"); - batches[i].set_destination_channel(channel); - batches[i].set_destination_node(i); - batches[i].set_type(MessageProto::TXN_BATCH); - } - - double time = GetTime(); - int txn_count = 0; - int batch_count = 0; - int batch_number = configuration_->this_node_id; - -#ifdef LATENCY_TEST - int watched_txn = -1; -#endif -if(deconstructor_invoked_ == true) -std::cout << "deconstructor_invoked_ is true.\n" << std::flush; - while (!deconstructor_invoked_) { - // Get batch from Paxos service. - string batch_string; - MessageProto batch_message; -#ifdef PAXOS - paxos.GetNextBatchBlocking(&batch_string); -#else - bool got_batch = false; - do { - pthread_mutex_lock(&mutex_); - if (batch_queue_.size()) { - batch_string = batch_queue_.front(); - batch_queue_.pop(); - got_batch = true; - } - pthread_mutex_unlock(&mutex_); - if (!got_batch) - Spin(0.001); - - } while (!got_batch); -#endif - batch_message.ParseFromString(batch_string); - for (int i = 0; i < batch_message.data_size(); i++) { - TxnProto txn; - txn.ParseFromString(batch_message.data(i)); - - - // Compute readers & writers; store in txn proto. - set readers; - set writers; - for (int i = 0; i < txn.read_set_size(); i++) - readers.insert(configuration_->LookupPartition(txn.read_set(i))); - for (int i = 0; i < txn.write_set_size(); i++) - writers.insert(configuration_->LookupPartition(txn.write_set(i))); - for (int i = 0; i < txn.read_write_set_size(); i++) { - writers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - readers.insert(configuration_->LookupPartition(txn.read_write_set(i))); - } - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) - txn.add_readers(*it); - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - txn.add_writers(*it); - bytes txn_data; - txn.SerializeToString(&txn_data); - - - for (set::iterator it = writers.begin(); it != writers.end(); ++it) - readers.insert(*it); - - - for (set::iterator it = readers.begin(); it != readers.end(); ++it) { - batches[*it].add_data(txn_data); - } - - txn_count++; - } - // Send this epoch's requests to all schedulers. - for (map::iterator it = batches.begin(); - it != batches.end(); ++it) { - it->second.set_batch_number(batch_number); - connection_->Send(it->second); - it->second.clear_data(); - } - batch_number += configuration_->all_nodes.size(); - batch_count++; - - // Report output. - if (GetTime() > time + 1) { - - // Reset txn count. - time = GetTime(); - txn_count = 0; - batch_count = 0; - } - } - Spin(1); -} diff --git a/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.h b/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.h deleted file mode 100755 index 3b75b3e9..00000000 --- a/src_traditional_2pl_2pc_4_partitions/sequencer/sequencer.h +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// Author: Kun Ren (kun@cs.yale.edu) -// -// The sequencer component of the system is responsible for choosing a global -// serial order of transactions to which execution must maintain equivalence. - -#ifndef _DB_SEQUENCER_SEQUENCER_H_ -#define _DB_SEQUENCER_SEQUENCER_H_ - -#include -#include -#include -#include - -//#define PAXOS -//#define PREFETCHING -//#define VERBOSE_SEQUENCER -#define COLD_CUTOFF 990000 - -#define MAX_BATCH_SIZE 25 - -#define SAMPLES 10000 -#define SAMPLE_RATE 999 - -//#define LATENCY_TEST1 - -using std::set; -using std::string; -using std::queue; -using std::tr1::unordered_map; - -class Configuration; -class Connection; -class Storage; -class TxnProto; - -#ifdef LATENCY_TEST1 -extern double execute_begin[SAMPLES]; -extern double execute_end[SAMPLES]; -#endif - -#ifdef LATENCY_TEST -extern double sequencer_recv[SAMPLES]; -// extern double paxos_begin[SAMPLES]; -// extern double paxos_end[SAMPLES]; -extern double sequencer_send[SAMPLES]; -extern double prefetch_cold[SAMPLES]; -extern double scheduler_lock[SAMPLES]; -extern double worker_begin[SAMPLES]; -extern double worker_end[SAMPLES]; -extern double scheduler_unlock[SAMPLES]; -#endif - -extern unordered_map doing_deadlocks; - -class Client { - public: - virtual ~Client() {} - virtual void GetTxn(TxnProto** txn, int txn_id) = 0; -}; - -class Sequencer { - public: - // The constructor creates background threads and starts the Sequencer's main - // loops running. - Sequencer(Configuration* conf, Connection* connection, Client* client); - - // Halts the main loops. - ~Sequencer(); - - private: - // Sequencer's main loops: - // - // RunWriter: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // Send batch to Paxos service. - // - // RunReader: - // while true: - // Spend epoch_duration collecting client txn requests into a batch. - // - // Executes in a background thread created and started by the constructor. - void RunWriter(); - void RunReader(); - - // Functions to start the Multiplexor's main loops, called in new pthreads by - // the Sequencer's constructor. - static void* RunSequencerWriter(void *arg); - static void* RunSequencerReader(void *arg); - - // Sets '*nodes' to contain the node_id of every node participating in 'txn'. - void FindParticipatingNodes(const TxnProto& txn, set* nodes); - - // Length of time spent collecting client requests before they are ordered, - // batched, and sent out to schedulers. - double epoch_duration_; - - // Configuration specifying node & system settings. - Configuration* configuration_; - - // Connection for sending and receiving protocol messages. - Connection* connection_; - - // Client from which to get incoming txns. - Client* client_; - - // Separate pthread contexts in which to run the sequencer's main loops. - pthread_t writer_thread_; - pthread_t reader_thread_; - - // False until the deconstructor is called. As soon as it is set to true, the - // main loop sees it and stops. - bool deconstructor_invoked_; - - // Queue for sending batches from writer to reader if not in paxos mode. - queue batch_queue_; - pthread_mutex_t mutex_; -}; -#endif // _DB_SEQUENCER_SEQUENCER_H_ diff --git a/src_traditional_2pl_2pc_4_partitions/tests/collapsed_versioned_storage_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/collapsed_versioned_storage_test.cc deleted file mode 100755 index 285b9134..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/collapsed_versioned_storage_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "backend/collapsed_versioned_storage.h" - -#include "common/testing.h" - -TEST(CollapsedVersionedStorageTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result = storage->ReadObject(key); - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 12)); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - EXPECT_TRUE(storage->PutObject(key, &value_one, 30)); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key, 12); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 20); - EXPECT_EQ(value_two, *result); - result = storage->ReadObject(key, 30); - EXPECT_EQ(value_one, *result); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 14)); - - EXPECT_EQ(0, storage->ReadObject(key, 12)); - result = storage->ReadObject(key); - EXPECT_EQ(value_one, *result); - - EXPECT_TRUE(storage->DeleteObject(key, 35)); - - delete storage; - - END; -} - -TEST(CheckpointingTest) { - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - - Key key = bytes("key"); - Value value_one = bytes("value_one"); - Value value_two = bytes("value_two"); - Value* result; - - EXPECT_TRUE(storage->PutObject(key, &value_one, 10)); - storage->PrepareForCheckpoint(15); - EXPECT_TRUE(storage->PutObject(key, &value_two, 20)); - storage->Checkpoint(); - - sleep(5); - - char checkpoint_path[100]; - snprintf(checkpoint_path, sizeof(checkpoint_path), "%s/15.checkpoint", - CHKPNTDIR); - FILE* checkpoint = fopen(checkpoint_path, "r"); - EXPECT_TRUE(checkpoint != NULL); - fclose(checkpoint); - - EXPECT_EQ(0, storage->ReadObject(key, 10)); - result = storage->ReadObject(key); - EXPECT_EQ(value_two, *result); - - delete storage; - - END; -} - -int main(int argc, char** argv) { - CollapsedVersionedStorageTest(); - CheckpointingTest(); -} - - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/configuration_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/configuration_test.cc deleted file mode 100755 index 79a569a5..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/configuration_test.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/configuration.h" - -#include "common/testing.h" - -// common/configuration_test.conf: -// # Node=:::: -// node1=0:1:16:128.36.232.50:50001 -// node2=0:2:16:128.36.232.50:50002 -TEST(ConfigurationTest_ReadFromFile) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(1, config.this_node_id); - EXPECT_EQ(2, config.all_nodes.size()); // 2 Nodes, node13 and node23. - EXPECT_EQ(1, config.all_nodes[1]->node_id); - EXPECT_EQ(50001, config.all_nodes[1]->port); - EXPECT_EQ(2, config.all_nodes[2]->node_id); - EXPECT_EQ(string("128.36.232.50"), config.all_nodes[2]->host); - END; -} - -// TODO(alex): Write proper test once partitioning is implemented. -TEST(ConfigurationTest_LookupPartition) { - Configuration config(1, "common/configuration_test.conf"); - EXPECT_EQ(0, config.LookupPartition(Key("0"))); - END; -} - -int main(int argc, char** argv) { - ConfigurationTest_ReadFromFile(); - ConfigurationTest_LookupPartition(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/connection_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/connection_test.cc deleted file mode 100755 index 8bf28f97..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/connection_test.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) -// -// TODO(alex): Write some tests spanning multiple physical machines. - -#include "common/connection.h" - -#include - -#include "common/testing.h" -/* -TEST(InprocTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(RemoteTest) { - Configuration config1(1, "common/configuration_test.conf"); - Configuration config2(2, "common/configuration_test.conf"); - - ConnectionMultiplexer* multiplexer1 = new ConnectionMultiplexer(&config1); - ConnectionMultiplexer* multiplexer2 = new ConnectionMultiplexer(&config2); - - Spin(0.1); - - Connection* c1 = multiplexer1->NewConnection("c1"); - Connection* c2 = multiplexer2->NewConnection("c2"); - - MessageProto message; - message.set_destination_node(2); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - c2->GetMessageBlocking(&message, 1); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -TEST(ChannelNotCreatedYetTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - - Spin(0.1); - - Connection* c1 = multiplexer->NewConnection("c1"); - - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c2"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - - c1->Send(message); - message.Clear(); - Spin(0.1); - - // Create channel after message is sent. - Connection* c2 = multiplexer->NewConnection("c2"); - c2->GetMessageBlocking(&message, 60); - - EXPECT_EQ("foo bar baz", message.data(0)); - - delete c1; - delete c2; - delete multiplexer; - - END; -} - -TEST(LinkUnlinkChannelTest) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Connection* c1 = multiplexer->NewConnection("c1"); - Connection* c2 = multiplexer->NewConnection("c2"); - c2->LinkChannel("c3"); - - Spin(0.1); - - // Send message to newly linked channel. - MessageProto message; - message.set_destination_node(0); - message.set_destination_channel("c3"); - message.set_type(MessageProto::EMPTY); - message.add_data("foo bar baz"); - c1->Send(message); - - // Receive message. - MessageProto m; - c2->GetMessageBlocking(&m, 60); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Send same message to channel before it is linked. - message.set_destination_channel("c4"); - c1->Send(message); - - Spin(0.1); - - // Recipient should not receive the message until linking the channel "c4". - EXPECT_FALSE(c2->GetMessage(&m)); - c2->LinkChannel("c4"); - Spin(0.1); // Give multiplexer time to link. - EXPECT_TRUE(c2->GetMessage(&m)); - EXPECT_EQ("foo bar baz", m.data(0)); - - // Unlink a channel and check that it no longer works. - c2->UnlinkChannel("c4"); - Spin(0.1); // Give multiplexer time to unlink. - c1->Send(message); - Spin(0.1); // Give multiplexer time to deliver the message. - EXPECT_FALSE(c2->GetMessage(&m)); - - // Deleting a connection should first free all its remaining links - // (i.e. "c3"). - delete c2; - Spin(0.1); // Give multiplexer time to unlink. - c2 = multiplexer->NewConnection("c3"); - message.set_destination_channel("c3"); - c1->Send(message); - c2->GetMessageBlocking(&m, 60); - - delete c1; - delete c2; - delete multiplexer; - - END; -} -*/ -int main(int argc, char** argv) { -// InprocTest(); -// RemoteTest(); -// ChannelNotCreatedYetTest(); -// LinkUnlinkChannelTest(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/deterministic_lock_manager_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/deterministic_lock_manager_test.cc deleted file mode 100755 index be34c6f8..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/deterministic_lock_manager_test.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/deterministic_lock_manager.h" - -#include -#include - -#include "applications/microbenchmark.h" -#include "applications/tpcc.h" -#include "common/utils.h" -#include "common/testing.h" - -using std::set; -/* -TEST(SimpleLockingTest) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - - // Txn 1 acquires read lock. - lm.Lock(key1, none, t1); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - - // Txn 2 requests write lock. Not granted. - lm.Lock(none, key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t3); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(1, ready_txns.size()); - - // Txn 1 releases lock. Txn 2 is granted write lock. - lm.Release(key1, t1); - EXPECT_EQ(WRITE, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t2, owners[0]); - EXPECT_EQ(2, ready_txns.size()); - EXPECT_EQ(t2, ready_txns.at(1)); - - // Txn 2 releases lock. Txn 3 is granted read lock. - lm.Release(key1, t2); - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(1, owners.size()); - EXPECT_EQ(t3, owners[0]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t3, ready_txns.at(2)); - - END; -} - -TEST(LocksReleasedOutOfOrder) { - deque ready_txns; - DeterministicLockManager lm(&ready_txns); - vector key1, none; - vector owners; - - key1.push_back(Key("key1")); - - TxnProto* t1 = (TxnProto*) 1; - TxnProto* t2 = (TxnProto*) 2; - TxnProto* t3 = (TxnProto*) 3; - TxnProto* t4 = (TxnProto*) 4; - - lm.Lock(key1, none, t1); // Txn 1 acquires read lock. - lm.Lock(none, key1, t2); // Txn 2 requests write lock. Not granted. - lm.Lock(key1, none, t3); // Txn 3 requests read lock. Not granted. - lm.Lock(key1, none, t4); // Txn 4 requests read lock. Not granted. - - lm.Release(key1, t2); // Txn 2 cancels write lock request. - - // Txns 1, 3 and 4 should now have a shared lock. - EXPECT_EQ(READ, lm.Status(Key("key1"), &owners)); - EXPECT_EQ(3, owners.size()); - EXPECT_EQ(t1, owners[0]); - EXPECT_EQ(t3, owners[1]); - EXPECT_EQ(t4, owners[2]); - EXPECT_EQ(3, ready_txns.size()); - EXPECT_EQ(t1, ready_txns.at(0)); - EXPECT_EQ(t3, ready_txns.at(1)); - EXPECT_EQ(t4, ready_txns.at(2)); - - END; -} -*/ - -TEST(ThroughputTest) { - deque ready_txns; - Configuration config(0, "common/configuration_test_one_node.conf"); - DeterministicLockManager lm(&ready_txns, &config); - vector txns; - - TPCC tpcc; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - for (int i = 0; i < 100000; i++) { -// txns.push_back(new TxnProto()); -// for (int j = 0; j < 10; j++) -// txns[i]->add_read_write_set(IntToString(j * 1000 + rand() % 1000)); - txns.push_back(tpcc.NewTxn(i, TPCC::NEW_ORDER, args_string, NULL)); - } - - double start = GetTime(); - - int next = 0; - for (int i = 0; i < 1000; i++) { - for (int j = 0; j < 100; j++) - lm.Lock(txns[next++]); - - while (ready_txns.size() > 0) { - TxnProto* txn = ready_txns.front(); - ready_txns.pop_front(); - lm.Release(txn); - } - } - - cout << 100000.0 / (GetTime() - start) << " txns/sec\n"; - - END; -} - -int main(int argc, char** argv) { -// SimpleLockingTest(); -// LocksReleasedOutOfOrder(); - ThroughputTest(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/deterministic_scheduler_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/deterministic_scheduler_test.cc deleted file mode 100755 index c0139888..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/deterministic_scheduler_test.cc +++ /dev/null @@ -1,10 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/determinstic_scheduler.h" - -#include - -#include "common/types.h" - -int main(int argc, char** argv) { -} diff --git a/src_traditional_2pl_2pc_4_partitions/tests/fetching_storage_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/fetching_storage_test.cc deleted file mode 100755 index 79a3e4ec..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/fetching_storage_test.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Author: Philip Shao (philip.shao@yale.edu) - -#include "backend/fetching_storage.h" - -#include -#include - -#include "common/testing.h" - - -TEST(FetchingStorageTest) { - system("rm ../db/storage/*"); - FetchingStorage* storage = FetchingStorage::BuildStorage(); - Key key = bytes("1"); - Value value = bytes("value"); - Value* result; - double wait_time; - EXPECT_FALSE(storage->Prefetch(key, &wait_time)); - EXPECT_TRUE(storage->PutObject(key, &value)); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - sleep(1); - EXPECT_TRUE(storage->Prefetch(key, &wait_time)); - sleep(1); - result = storage->ReadObject(key); - EXPECT_EQ(value, *result); - EXPECT_TRUE(storage->Unfetch(key)); - END; -} - -int main(int argc, char** argv) { - FetchingStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/microbenchmark_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/microbenchmark_test.cc deleted file mode 100755 index 14b5b8fd..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/microbenchmark_test.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "applications/microbenchmark.h" - -#include - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -using std::set; - -SimpleStorage* actual_storage; -Configuration* config; - -#define CHECK_OBJECT(KEY, EXPECTED_VALUE) do { \ - Value* actual_value; \ - actual_value = actual_storage->ReadObject(KEY); \ - EXPECT_EQ(EXPECTED_VALUE, *actual_value); \ -} while (0) - -TEST(MicrobenchmarkTest) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(config); - Connection* connection = multiplexer->NewConnection("asdf"); - actual_storage = new SimpleStorage(); - Microbenchmark microbenchmark(1, 100); - - // Initialize storage. - microbenchmark.InitializeStorage(actual_storage, config); - - // Execute a 'MICROTXN_SP' txn. - TxnProto* txn = microbenchmark.MicroTxnSP(1, 0); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* storage = new StorageManager(config, connection, - actual_storage, txn); - microbenchmark.Execute(txn, storage); - - // Check post-execution storage state. - set write_set; - for (int i = 0; i < Microbenchmark::kRWSetSize; i++) - write_set.insert(StringToInt(txn->write_set(i))); - for (int i = 0; i < microbenchmark.kDBSize; i++) { - if (write_set.count(i)) - CHECK_OBJECT(IntToString(i), IntToString(i+1)); - else - CHECK_OBJECT(IntToString(i), IntToString(i)); - } - - - - delete storage; - delete txn; - - delete actual_storage; - delete connection; - delete multiplexer; - delete config; - - END; -} - -int main(int argc, char** argv) { - MicrobenchmarkTest(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/paxos_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/paxos_test.cc deleted file mode 100755 index 20bdfe09..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/paxos_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Author: Kun Ren (kun.ren@yale.edu) - -#include "paxos/paxos.h" - -#include - -#include "common/testing.h" -#include "sequencer/sequencer.h" - -using std::vector; - -// Record the time of the batches submitted. -vector submit_time; -// Record the time of the batches received. -vector receive_time; -// The throuput of get batches from zookeeper. -int throughput; - -// Create Paxos object to submit some batches, inserting them into -// a globally consistent batch order. -void* Writer(void *arg) { - // Create paxos object. - Paxos writer(ZOOKEEPER_CONF, false); - - string test("test"); - double start, end; - int write_number = 0; - // Firstly, submit one batch, sleep for a while (random from 0 to 1 msec), - // submit another batch, for 70 seconds in all. - start = GetTime(); - while (1) { - // Submit the bach. - writer.SubmitBatch(test); - end = GetTime(); - submit_time.push_back(end); - write_number++; - // The interal is 0 to 1 msec. - srand(50); - usleep((rand()%10)*100); - // Test for 70 seconds. - if (end - start > 70) - break; - } - sleep(30); - return NULL; -} - -// Create Paxos object to read batches from zookeeper. -void* Reader(void *arg) { - // Create Paxos object. - Paxos reader(ZOOKEEPER_CONF, true); - - string batch_data; - double start, end; - int read_number = 0; - // Continued get batches from zookeeper server. - start = GetTime(); - while (1) { - bool rc = reader.GetNextBatch(&batch_data); - if (rc == true) { - read_number++; - end = GetTime(); - receive_time.push_back(end); - // If have received around 200k batches, break the loop, - // record the running time. - if (read_number >= 200000) { - end = GetTime(); - throughput = 200000 / static_cast(end - start); - break; - } - } else { - end = GetTime(); - // Timeout is 150 sec. - if (end - start > 150) { - printf("Writer writes no more than 20k batches.\n"); - break; - } - } - } - sleep(20); - return NULL; -} - -TEST(PaxosTest) { - printf("Running zookeeper test.\n"); - - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, Writer, NULL); - pthread_create(&thread_2, NULL, Reader, NULL); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - // Compute the average latency. - double sum_latency = 0; - for (unsigned int i = 0; i < receive_time.size(); i++) { - sum_latency += (receive_time[i] - submit_time[i]); - } - double average_latency = sum_latency * 1000 / receive_time.size(); - - printf("Throughput: %d txns/sec.\n", throughput); - printf("Average latency: %lf ms.\n", average_latency); - - printf("Running zookeeper test.......done.\n"); - END; -} - -int main(int argc, char** argv) { -#ifdef PAXOS - // Start zookeeper - // StartZookeeper(ZOOKEEPER_CONF); - // printf("Starting zookeeper servers.......done.\n"); - // Run zookeeper test - PaxosTest(); - // Stop zookeeper -// printf("Stopping zookeeper servers.\n"); -// StopZookeeper(ZOOKEEPER_CONF); -// printf("Stopping zookeeper servers.......done.\n"); -#endif -} diff --git a/src_traditional_2pl_2pc_4_partitions/tests/profile_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/profile_test.cc deleted file mode 100755 index f55e19f0..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/profile_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "applications/tpcc.h" -#include "backend/collapsed_versioned_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -int main(int argc, char** argv) { - Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); - CollapsedVersionedStorage* storage = new CollapsedVersionedStorage(); - TPCC* tpcc = new TPCC(); - - TPCC().InitializeStorage(storage, config); - - for (int i = 0; i < 100000; i++) { - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - TxnProto* txn = tpcc->NewTxn(0, TPCC::NEW_ORDER, args_string, config); - txn->add_readers(0); - txn->add_writers(0); - - StorageManager* manager = new StorageManager(config, NULL, storage, txn); - - tpcc->Execute(txn, manager); - - delete manager; - delete txn; - } - - delete tpcc; - delete storage; - delete config; -} diff --git a/src_traditional_2pl_2pc_4_partitions/tests/sequencer_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/sequencer_test.cc deleted file mode 100755 index 12c5b6b9..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/sequencer_test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "sequencer/sequencer.h" - -#include - -#include "common/testing.h" - -TEST(SequencerTest) { - END; -} - -int main(int argc, char** argv) { - SequencerTest(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/serial_scheduler_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/serial_scheduler_test.cc deleted file mode 100755 index 8ce8054c..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/serial_scheduler_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "scheduler/serial_scheduler.h" - -#include "common/testing.h" - -TEST(SerialSchedulerTest) { - END; -} - -int main() { - SerialSchedulerTest(); -} diff --git a/src_traditional_2pl_2pc_4_partitions/tests/simple_storage_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/simple_storage_test.cc deleted file mode 100755 index 6a855366..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/simple_storage_test.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/simple_storage.h" - -#include "common/testing.h" - -TEST(SimpleStorageTest) { - SimpleStorage storage; - Key key = bytes("key"); - Value value = bytes("value"); - Value* result; - EXPECT_EQ(0, storage.ReadObject(key)); - EXPECT_TRUE(storage.PutObject(key, &value)); - result = storage.ReadObject(key); - EXPECT_EQ(value, *result); - - EXPECT_TRUE(storage.DeleteObject(key)); - EXPECT_EQ(0, storage.ReadObject(key)); - - END; -} - -int main(int argc, char** argv) { - SimpleStorageTest(); -} - - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/storage_manager_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/storage_manager_test.cc deleted file mode 100755 index be719e5c..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/storage_manager_test.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "backend/storage_manager.h" - -#include - -#include "backend/simple_storage.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" -#include "proto/txn.pb.h" - -TEST(SingleNode) { - Configuration config(0, "common/configuration_test_one_node.conf"); - ConnectionMultiplexer* multiplexer = new ConnectionMultiplexer(&config); - Spin(0.1); - Connection* connection = multiplexer->NewConnection("storage_manager"); - SimpleStorage storage; - - string a = "a"; - string c = "c"; - storage.PutObject("0", &a); - storage.PutObject("2", &c); - TxnProto txn; - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_write_set("2"); - txn.add_readers(1); - txn.add_writers(1); - - StorageManager* storage_manager = - new StorageManager(&config, connection, &storage, &txn); - - Value* result_x; - result_x = storage_manager->ReadObject("0"); - EXPECT_TRUE(storage_manager->PutObject("2", result_x)); - - result_x = storage.ReadObject("2"); - EXPECT_EQ("a", *result_x); - - delete storage_manager; - delete connection; - delete multiplexer; - - END; -} - -// TODO(alex): Global variables of class type are prohibited by the google style -// guide. -Configuration config1(1, "common/configuration_test.conf"); -Configuration config2(2, "common/configuration_test.conf"); -ConnectionMultiplexer* multiplexer1; -ConnectionMultiplexer* multiplexer2; -Connection* c1; -Connection* c2; -SimpleStorage storage1; -SimpleStorage storage2; -TxnProto txn; - -void* ExecuteTxn(void* arg) { - int node = *reinterpret_cast(arg); - - StorageManager* manager; - if (node == 1) - manager = new StorageManager(&config1, c1, &storage1, &txn); - else - manager = new StorageManager(&config2, c2, &storage2, &txn); - - Value* result_x; - Value* result_xy; - result_x = manager->ReadObject("0"); - result_xy = manager->ReadObject("1"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - EXPECT_TRUE(manager->PutObject("2", result_x)); - EXPECT_TRUE(manager->PutObject("3", result_xy)); - result_x = manager->ReadObject("2"); - result_xy = manager->ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete manager; - - return NULL; -} - -TEST(TwoNodes) { - multiplexer1 = new ConnectionMultiplexer(&config1); - multiplexer2 = new ConnectionMultiplexer(&config2); - Spin(0.1); - c1 = multiplexer1->NewConnection("1"); - c2 = multiplexer2->NewConnection("1"); - - string a = "a"; - string b = "b"; - string c = "c"; - string d = "d"; - storage1.PutObject("0", &a); - storage2.PutObject("1", &b); - storage1.PutObject("2", &c); - storage2.PutObject("3", &d); - txn.set_txn_id(1); - txn.add_read_set("0"); - txn.add_read_set("1"); - txn.add_write_set("2"); - txn.add_write_set("3"); - txn.add_readers(1); - txn.add_readers(2); - txn.add_writers(1); - txn.add_writers(2); - - int node1 = 1; - int node2 = 2; - pthread_t thread_1; - pthread_t thread_2; - pthread_create(&thread_1, NULL, ExecuteTxn, reinterpret_cast(&node1)); - pthread_create(&thread_2, NULL, ExecuteTxn, reinterpret_cast(&node2)); - pthread_join(thread_1, NULL); - pthread_join(thread_2, NULL); - - Value* result_x; - Value* result_xy; - result_x = storage1.ReadObject("2"); - result_xy = storage2.ReadObject("3"); - EXPECT_EQ("a", *result_x); - EXPECT_EQ("b", *result_xy); - - delete c1; - delete c2; - delete multiplexer1; - delete multiplexer2; - - END; -} - -int main(int argc, char** argv) { -// TODO(alex): Fix these tests! -// SingleNode(); -// TwoNodes(); -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/tpcc_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/tpcc_test.cc deleted file mode 100755 index 8dff4592..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/tpcc_test.cc +++ /dev/null @@ -1,577 +0,0 @@ -// Author: Thaddeus Diamond (diamond@cs.yale.edu) - -#include "applications/tpcc.h" - -#include "backend/simple_storage.h" -#include "backend/storage_manager.h" -#include "common/configuration.h" -#include "common/connection.h" -#include "common/testing.h" -#include "common/utils.h" - -// We make these global variables to avoid weird pointer passing and code -// redundancy -Configuration* config = - new Configuration(0, "common/configuration_test_one_node.conf"); -ConnectionMultiplexer* multiplexer; -Connection* connection; -SimpleStorage* simple_store; -TPCC* tpcc; - -// Test the id generation -TEST(IdGenerationTest) { - EXPECT_EQ(tpcc->CheckpointID("w1"), 1); - EXPECT_EQ(tpcc->CheckpointID("d1"), WAREHOUSES_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("c1"), WAREHOUSES_PER_NODE + - DISTRICTS_PER_NODE + 1); - EXPECT_EQ(tpcc->CheckpointID("w2si1"), - 1000000 + 2 * NUMBER_OF_ITEMS + 1); - EXPECT_EQ(tpcc->CheckpointID("i1"), 3000001); - EXPECT_EQ(tpcc->CheckpointID("ol1"), 4000001); - EXPECT_EQ(tpcc->CheckpointID("no1"), 5000001); - EXPECT_EQ(tpcc->CheckpointID("o1"), 6000001); - EXPECT_EQ(tpcc->CheckpointID("h1"), 7000001); - EXPECT_EQ(tpcc->CheckpointID("ln1"), 8000001); - - END -} - -// Test for creation of a warehouse, ensure the attributes are correct -TEST(WarehouseTest) { - Warehouse* warehouse = tpcc->CreateWarehouse("w1"); - - EXPECT_EQ(warehouse->id(), "w1"); - EXPECT_TRUE(warehouse->has_name()); - EXPECT_TRUE(warehouse->has_street_1()); - EXPECT_TRUE(warehouse->has_street_2()); - EXPECT_TRUE(warehouse->has_city()); - EXPECT_TRUE(warehouse->has_state()); - EXPECT_TRUE(warehouse->has_zip()); - EXPECT_EQ(warehouse->tax(), 0.05); - EXPECT_EQ(warehouse->year_to_date(), 0.0); - - // Finish - delete warehouse; - END -} - -// Test for creation of a district, ensure the attributes are correct -TEST(DistrictTest) { - District* district = tpcc->CreateDistrict("d1", "w1"); - - EXPECT_EQ(district->id(), "d1"); - EXPECT_EQ(district->warehouse_id(), "w1"); - EXPECT_TRUE(district->has_name()); - EXPECT_TRUE(district->has_street_1()); - EXPECT_TRUE(district->has_street_2()); - EXPECT_TRUE(district->has_city()); - EXPECT_TRUE(district->has_state()); - EXPECT_TRUE(district->has_zip()); - EXPECT_EQ(district->tax(), 0.05); - EXPECT_EQ(district->year_to_date(), 0.0); - EXPECT_EQ(district->next_order_id(), 1); - - // Finish - delete district; - END -} - -// Test for creation of a customer, ensure the attributes are correct -TEST(CustomerTest) { - // Create a transaction so the customer creation can do secondary insertion - TxnProto* secondary_keying = new TxnProto(); - secondary_keying->set_txn_id(1); - Customer* customer = tpcc->CreateCustomer("c1", "d1", "w1"); - - EXPECT_EQ(strcmp(customer->id().c_str(), "c1"), 0); - EXPECT_EQ(strcmp(customer->district_id().c_str(), "d1"), 0); - EXPECT_EQ(strcmp(customer->warehouse_id().c_str(), "w1"), 0); - EXPECT_TRUE(customer->has_first()); - EXPECT_TRUE(customer->has_middle()); - EXPECT_TRUE(customer->has_last()); - EXPECT_TRUE(customer->has_street_1()); - EXPECT_TRUE(customer->has_street_2()); - EXPECT_TRUE(customer->has_city()); - EXPECT_TRUE(customer->has_state()); - EXPECT_TRUE(customer->has_zip()); - EXPECT_TRUE(customer->has_data()); - EXPECT_EQ(customer->since(), 0); - EXPECT_EQ(customer->credit(), "GC"); - EXPECT_EQ(customer->credit_limit(), 0.01); - EXPECT_EQ(customer->discount(), 0.5); - EXPECT_EQ(customer->balance(), 0); - EXPECT_EQ(customer->year_to_date_payment(), 0); - EXPECT_EQ(customer->payment_count(), 0); - EXPECT_EQ(customer->delivery_count(), 0); - - // Finish - delete secondary_keying; - delete customer; - END -} - -// Test for creation of an item, ensure the attributes are correct -TEST(ItemTest) { - Item* item = tpcc->CreateItem("i1"); - - EXPECT_EQ(item->id(), "i1"); - EXPECT_TRUE(item->has_name()); - EXPECT_TRUE(item->has_price()); - EXPECT_TRUE(item->has_data()); - - // Finish - delete item; - END -} - -// Test for creation of a stock, ensure the attributes are correct -TEST(StockTest) { - Stock* stock = tpcc->CreateStock("i1", "w1"); - - EXPECT_EQ(stock->id(), "w1si1"); - EXPECT_EQ(stock->warehouse_id(), "w1"); - EXPECT_EQ(stock->item_id(), "i1"); - EXPECT_TRUE(stock->has_quantity()); - EXPECT_TRUE(stock->has_data()); - EXPECT_EQ(stock->year_to_date(), 0); - EXPECT_EQ(stock->order_count(), 0); - EXPECT_EQ(stock->remote_count(), 0); - - // Finish - delete stock; - END -} - -// This initializes a new transaction and ensures it has the desired properties -TEST(NewTxnTest) { - TPCCArgs* tpcc_args = new TPCCArgs(); - tpcc_args->set_system_time(GetTime()); - - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Initialize Transaction Generation - TxnProto* txn = tpcc->NewTxn(1, TPCC::INITIALIZE, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 1); - EXPECT_EQ(txn->txn_type(), TPCC::INITIALIZE); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - // New Order Transaction Generation - delete txn; - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 2); - EXPECT_EQ(txn->txn_type(), TPCC::NEW_ORDER); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->order_line_count() >= 5 && - tpcc_args->order_line_count() <= 15); - EXPECT_TRUE(txn->write_set_size() == tpcc_args->order_line_count() + 2); - for (int i = 0; i < tpcc_args->order_line_count(); i++) - EXPECT_TRUE(tpcc_args->quantities(i) <= 10 && tpcc_args->quantities(i) > 0); - - // Payment Transaction Generation - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - EXPECT_EQ(txn->txn_id(), 4); - EXPECT_EQ(txn->txn_type(), TPCC::PAYMENT); - EXPECT_EQ(txn->isolation_level(), TxnProto::SERIALIZABLE); - EXPECT_EQ(txn->status(), TxnProto::NEW); - - EXPECT_TRUE(tpcc_args->ParseFromString(txn->arg())); - EXPECT_TRUE(tpcc_args->amount() >= 1 && tpcc_args->amount() <= 5000); - EXPECT_EQ(txn->write_set_size(), 1); - EXPECT_TRUE(txn->read_set_size() == 0 || txn->read_set_size() == 1); - - // Finish - delete txn; - delete txn_args; - delete tpcc_args; - END -} - -// Initialize the database and ensure that there are the correct -// objects actually in the database -TEST(InitializeTest) { - // Run initialization method. - tpcc->InitializeStorage(simple_store, config); - - // Expect all the warehouses to be there - for (int i = 0; i < WAREHOUSES_PER_NODE; i++) { - char warehouse_key[128]; - Value* warehouse_value; - snprintf(warehouse_key, sizeof(warehouse_key), "w%d", i); - warehouse_value = simple_store->ReadObject(warehouse_key); - - Warehouse* dummy_warehouse = new Warehouse(); - EXPECT_TRUE(dummy_warehouse->ParseFromString(*warehouse_value)); - delete dummy_warehouse; - - // Expect all the districts to be there - for (int j = 0; j < DISTRICTS_PER_WAREHOUSE; j++) { - char district_key[128]; - Value* district_value; - snprintf(district_key, sizeof(district_key), "w%dd%d", - i, j); - district_value = simple_store->ReadObject(district_key); - - District* dummy_district = new District(); - EXPECT_TRUE(dummy_district->ParseFromString(*district_value)); - delete dummy_district; - - // Expect all the customers to be there - for (int k = 0; k < CUSTOMERS_PER_DISTRICT; k++) { - char customer_key[128]; - Value* customer_value; - snprintf(customer_key, sizeof(customer_key), - "w%dd%dc%d", i, j, k); - customer_value = simple_store->ReadObject(customer_key); - - Customer* dummy_customer = new Customer(); - EXPECT_TRUE(dummy_customer->ParseFromString(*customer_value)); - delete dummy_customer; - } - } - - // Expect all stock to be there - for (int j = 0; j < NUMBER_OF_ITEMS; j++) { - char item_key[128], stock_key[128]; - Value* stock_value; - snprintf(item_key, sizeof(item_key), "i%d", j); - snprintf(stock_key, sizeof(stock_key), "%ss%s", - warehouse_key, item_key); - stock_value = simple_store->ReadObject(stock_key); - - Stock* dummy_stock = new Stock(); - EXPECT_TRUE(dummy_stock->ParseFromString(*stock_value)); - delete dummy_stock; - } - } - - // Expect all items to be there - for (int i = 0; i < NUMBER_OF_ITEMS; i++) { - char item_key[128]; - Value item_value; - snprintf(item_key, sizeof(item_key), "i%d", i); - item_value = *(tpcc->GetItem(string(item_key))); - - Item* dummy_item = new Item(); - EXPECT_TRUE(dummy_item->ParseFromString(item_value)); - delete dummy_item; - } - - END; -} - -TEST(NewOrderTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm new orders are satisfying TPC-C standards - TxnProto* txn; - bool invalid; - do { - txn = tpcc->NewTxn(2, TPCC::NEW_ORDER, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - invalid = false; - for (int i = 0; i < txn_args->order_line_count(); i++) { - if (txn->read_write_set(i + 1).find("i-1") != string::npos) - invalid = true; - } - } while (invalid); - - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(0)); - assert(district->ParseFromString(*district_value)); - - // Prefetch the stocks - Stock* old_stocks[txn_args->order_line_count()]; - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - assert(stock->ParseFromString(*stock_value)); - old_stocks[i] = stock; - } - - // Prefetch the actual values - int old_next_order_id = district->next_order_id(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Let's prefetch the keys we need for the post-check - Key district_key = txn->read_write_set(0); - Key new_order_key = txn->write_set(txn_args->order_line_count()); - Key order_key = txn->write_set(txn_args->order_line_count() + 1); - - // Add in all the keys and re-initialize the storage manager - txn->add_read_set(new_order_key); - txn->add_read_set(order_key); - for (int i = 0; i < txn_args->order_line_count(); i++) { - txn->add_read_set(txn->write_set(i)); - } - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - // Ensure that D_NEXT_O_ID is incremented for district - district_value = storage->ReadObject(district_key); - assert(district->ParseFromString(*district_value)); - EXPECT_EQ(old_next_order_id + 1, district->next_order_id()); - - // TPCC::NEW_ORDER row was inserted with appropriate fields - Value* new_order_value; - NewOrder* new_order = new NewOrder(); - new_order_value = storage->ReadObject(new_order_key); - EXPECT_TRUE(new_order->ParseFromString(*new_order_value)); - - // ORDER row was inserted with appropriate fields - Value* order_value; - Order* order = new Order(); - order_value = storage->ReadObject(order_key); - EXPECT_TRUE(order->ParseFromString(*order_value)); - - // For each item in O_OL_CNT - for (int i = 0; i < txn_args->order_line_count(); i++) { - Value* stock_value; - Stock* stock = new Stock(); - stock_value = storage->ReadObject(txn->read_write_set(i + 1)); - EXPECT_TRUE(stock->ParseFromString(*stock_value)); - - // Check YTD, order_count, and remote_count - int corrected_year_to_date = old_stocks[i]->year_to_date(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_year_to_date += txn_args->quantities(j); - } - EXPECT_EQ(stock->year_to_date(), corrected_year_to_date); - - // Check order_count - int corrected_order_count = old_stocks[i]->order_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_order_count--; - } - EXPECT_EQ(stock->order_count(), corrected_order_count); - - // Check remote_count - if (txn->multipartition()) { - int corrected_remote_count = old_stocks[i]->remote_count(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) - corrected_remote_count++; - } - EXPECT_EQ(stock->remote_count(), corrected_remote_count); - } - - // Check stock supply decrease - int corrected_quantity = old_stocks[i]->quantity(); - for (int j = 0; j < txn_args->order_line_count(); j++) { - if (txn->read_write_set(j + 1) == txn->read_write_set(i + 1)) { - if (old_stocks[i]->quantity() >= txn_args->quantities(i) + 10) - corrected_quantity -= txn_args->quantities(j); - else - corrected_quantity -= txn_args->quantities(j) - 91; - } - } - EXPECT_EQ(stock->quantity(), corrected_quantity); - - // First, we check if the item is valid - size_t item_idx = txn->read_write_set(i + 1).find("i"); - Key item_key = txn->read_write_set(i + 1).substr(item_idx, string::npos); - Value item_value = *(tpcc->GetItem(item_key)); - Item* item = new Item(); - EXPECT_TRUE(item->ParseFromString(item_value)); - - // Check the order line - // TODO(Thad): Get order_line_ptr from Order protobuf and deserialize from - // there -// Value* order_line_value; -// OrderLine* order_line = new OrderLine(); -// order_line_value = storage->ReadObject(txn->write_set(i)); -// EXPECT_TRUE(order_line->ParseFromString(*order_line_value)); -// EXPECT_EQ(order_line->amount(), item->price() * txn_args->quantities(i)); -// EXPECT_EQ(order_line->number(), i); - - // Free memory -// delete order_line; - delete item; - delete stock; - } - - // Free memory - for (int i = 0; i < txn_args->order_line_count(); i++) - delete old_stocks[i]; - delete txn_args; - delete storage; - delete district; - delete order; - delete new_order; - delete txn; - - END -} - -TEST(PaymentTest) { - // Txn args w/number of warehouses - TPCCArgs* txn_args = new TPCCArgs(); - txn_args->set_multipartition(false); - txn_args->set_system_time(GetTime()); - Value txn_args_value; - assert(txn_args->SerializeToString(&txn_args_value)); - - // Do work here to confirm payment transactions are satisfying standards - TxnProto* txn = new TxnProto(); - do { - delete txn; - txn = tpcc->NewTxn(4, TPCC::PAYMENT, txn_args_value, config); - assert(txn_args->ParseFromString(txn->arg())); - } while (txn->read_write_set_size() < 3); - txn->add_read_set(txn->write_set(0)); - txn->add_readers(0); - txn->add_writers(0); - StorageManager* storage = new StorageManager(config, connection, simple_store, - txn); - - // Prefetch some values in order to ensure our ACIDity after - Warehouse *warehouse = new Warehouse(); - Value* warehouse_value; - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - int old_warehouse_year_to_date = warehouse->year_to_date(); - - // Prefetch district - District *district = new District(); - Value* district_value; - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - int old_district_year_to_date = district->year_to_date(); - - // Preetch customer - Customer *customer = new Customer(); - Value* customer_value; - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - int old_customer_year_to_date_payment = customer->year_to_date_payment(); - int old_customer_balance = customer->balance(); - int old_customer_payment_count = customer->payment_count(); - - // Execute the transaction - tpcc->Execute(txn, storage); - - // Get the data back from the database - delete storage; - storage = new StorageManager(config, connection, simple_store, txn); - - warehouse_value = storage->ReadObject(txn->read_write_set(0)); - assert(warehouse->ParseFromString(*warehouse_value)); - district_value = storage->ReadObject(txn->read_write_set(1)); - assert(district->ParseFromString(*district_value)); - customer_value = storage->ReadObject(txn->read_write_set(2)); - assert(customer->ParseFromString(*customer_value)); - - // Check the old values against the new - EXPECT_EQ(warehouse->year_to_date(), old_warehouse_year_to_date + - txn_args->amount()); - EXPECT_EQ(district->year_to_date(), old_district_year_to_date + - txn_args->amount()); - EXPECT_EQ(customer->year_to_date_payment(), - old_customer_year_to_date_payment + txn_args->amount()); - EXPECT_EQ(customer->balance(), old_customer_balance - txn_args->amount()); - EXPECT_EQ(customer->payment_count(), old_customer_payment_count + 1); - - // Ensure the history record is valid - History* history = new History(); - Value* history_value; - history_value = storage->ReadObject(txn->read_set(0)); - EXPECT_TRUE(history->ParseFromString(*history_value)); - EXPECT_EQ(history->warehouse_id(), warehouse->id()); - EXPECT_EQ(history->district_id(), district->id()); - EXPECT_EQ(history->customer_id(), customer->id()); - EXPECT_EQ(history->customer_warehouse_id(), customer->warehouse_id()); - EXPECT_EQ(history->customer_district_id(), customer->district_id()); - - // Free memory - delete history; - delete warehouse; - delete customer; - delete district; - delete storage; - delete txn_args; - delete txn; - - END -} - -TEST(MultipleTxnTest) { - StorageManager* storage; - TPCCArgs args; - args.set_system_time(GetTime()); - args.set_multipartition(false); - string args_string; - args.SerializeToString(&args_string); - - TxnProto* txn = tpcc->NewTxn(0, TPCC::INITIALIZE, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - txn = tpcc->NewTxn(1, TPCC::NEW_ORDER, args_string, config); - storage = new StorageManager(config, connection, simple_store, txn); - tpcc->Execute(txn, storage); - delete storage; - delete txn; - - END; -} - -int main(int argc, char** argv) { - config = new Configuration(0, "common/configuration_test_one_node.conf"); - multiplexer = new ConnectionMultiplexer(config); - connection = multiplexer->NewConnection("asdf"); - simple_store = new SimpleStorage(); - tpcc = new TPCC(); - - InitializeTest(); - - IdGenerationTest(); - - WarehouseTest(); - DistrictTest(); - CustomerTest(); - ItemTest(); - StockTest(); - - NewTxnTest(); - NewOrderTest(); - PaymentTest(); - - // MultipleTxnTest(); - - delete tpcc; - delete simple_store; - delete connection; - delete multiplexer; - delete config; - - return 0; -} - diff --git a/src_traditional_2pl_2pc_4_partitions/tests/types_test.cc b/src_traditional_2pl_2pc_4_partitions/tests/types_test.cc deleted file mode 100755 index ef5bc102..00000000 --- a/src_traditional_2pl_2pc_4_partitions/tests/types_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Author: Alexander Thomson (thomson@cs.yale.edu) - -#include "common/types.h" - -#include "common/testing.h" - -TEST(PackSignedIntTest) { - int8 i1 = 65; - int16 i2 = -2551; - int32 i3 = 0; - int64 i4 = -2551255125512551; - - EXPECT_EQ(i1, UnpackInt8(PackInt8(i1))); - EXPECT_EQ(i2, UnpackInt16(PackInt16(i2))); - EXPECT_EQ(i3, UnpackInt32(PackInt32(i3))); - EXPECT_EQ(i4, UnpackInt64(PackInt64(i4))); - - END; -} - -TEST(PackUnsignedIntTest) { - uint8 u1 = 251; - uint16 u2 = 2551; - uint32 u3 = 0; - uint64 u4 = 2551255125512551; - - EXPECT_EQ(u1, UnpackUInt8(PackUInt8(u1))); - EXPECT_EQ(u2, UnpackUInt16(PackUInt16(u2))); - EXPECT_EQ(u3, UnpackUInt32(PackUInt32(u3))); - EXPECT_EQ(u4, UnpackUInt64(PackUInt64(u4))); - - END; -} - -int main(int argc, char** argv) { - PackSignedIntTest(); - PackUnsignedIntTest(); -}