Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JVM Snapshot Implementation #150

Open
wants to merge 51 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
bc94464
checkpoint 1
gegao Oct 17, 2013
175a85c
Fork version snapshot checkpoint
gegao Oct 20, 2013
2e8baba
working prototype
gegao Oct 22, 2013
ab31b64
Transaction serialization
gegao Oct 23, 2013
ed62b3c
Runnable from eclipse; add Junit test ( can't pass )
gegao Nov 11, 2013
ad9847c
minor change
gegao Nov 11, 2013
0baeeef
pass test on AWS
gegao Nov 11, 2013
6821326
some cleanup
gegao Nov 11, 2013
f18bd45
debug
gegao Nov 11, 2013
5c5bc36
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 11, 2013
b9cd7e2
add catch connection failure in the parent code
gegao Nov 12, 2013
ca9cc2b
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 12, 2013
e734982
add test for JVMSnapshotSuite
gegao Nov 12, 2013
2a7b877
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 12, 2013
100cf0b
minor bug fix
gegao Nov 12, 2013
82408e6
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 12, 2013
f463056
debug
gegao Nov 13, 2013
45c50fb
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 13, 2013
7a2f819
debug
gegao Nov 13, 2013
99bfb38
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 13, 2013
9d4c519
debug
gegao Nov 14, 2013
1afeecf
clean up debug output, set up snapshot refresh and snapshot kill
gegao Nov 14, 2013
9ab099b
minor bug
gegao Nov 14, 2013
65443ef
add network retry
gegao Nov 14, 2013
1f5ef0b
minor
gegao Nov 14, 2013
2a82b72
Change kill method
gegao Nov 14, 2013
a63b2ba
modify eventloop init
gegao Nov 14, 2013
453c209
Add main thread in HStoreJVMSnapshotManager; Add queue; Add two more …
gegao Nov 17, 2013
e6a0e95
minor change about NIOEventloop
gegao Nov 17, 2013
bc4bf1e
Snapshot notify wait
gegao Nov 17, 2013
caec9fa
Merge branch 'master' of github.com:gegao/h-store
gegao Nov 17, 2013
3ea3825
Change to block rpc
gegao Nov 17, 2013
5389c27
shutdown request
gegao Nov 17, 2013
591ad52
Merge branch 'master' of https://github.com/gegao/h-store
gegao Nov 17, 2013
690cd1d
block rpc
gegao Nov 17, 2013
da85e0f
Socket connection prototype
gegao Nov 18, 2013
20eded8
Code clean up. Use socket communication
gegao Nov 18, 2013
0a0be7d
add debug output
gegao Nov 18, 2013
5043fc5
debug
gegao Nov 19, 2013
92e3624
Add send response for dtxn
gegao Nov 19, 2013
42ddae8
More debug
gegao Nov 19, 2013
94fa735
remove sleep
gegao Nov 25, 2013
03536a1
add back sleep
gegao Nov 25, 2013
6d7a274
change the way of exiting
gegao Nov 25, 2013
150756a
final v
Dec 3, 2013
527f127
try
Dec 3, 2013
5c09ba2
code clean up
gegao Dec 31, 2013
0b65b9e
code clean up 2
gegao Dec 31, 2013
dcde697
code merge
gegao Dec 31, 2013
14a51e0
code clean up 3
gegao Dec 31, 2013
9a08f0e
code clean up 4
gegao Dec 31, 2013
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ SourceJars
src/ee/org_voltdb_jni_ExecutionEngine.h
src/ee/org_voltdb_utils_DBBPool.h
src/ee/org_voltdb_utils_ThreadUtils.h
src/ee/org_voltdb_utils_ProcessUtils.h

## -------------------------------
## Catalog Compiler Artifacts
Expand Down
3 changes: 3 additions & 0 deletions build-common.xml
Original file line number Diff line number Diff line change
Expand Up @@ -215,4 +215,7 @@
<arg value="site.pool_scale_factor=${site.pool_scale_factor}" />
<arg value="site.pool_profiling=${site.pool_profiling}" />
<arg value="site.pool_pathestimators_idle=${site.pool_pathestimators_idle}" />
<arg value="site.jvmsnapshot_enable=${site.jvmsnapshot_enable}" />
<arg value="site.jvmsnapshot_interval=${site.jvmsnapshot_interval}" />
<arg value="site.jvmsnapshot_start=${site.jvmsnapshot_start}" />

18 changes: 18 additions & 0 deletions build.xml
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,7 @@ NATIVE EE STUFF
<delete file="${src.ee.dir}/org_voltdb_jni_ExecutionEngine.h" />
<delete file="${src.ee.dir}/org_voltdb_utils_DBBPool.h" />
<delete file="${src.ee.dir}/org_voltdb_utils_ThreadUtils.h" />
<delete file="${src.ee.dir}/org_voltdb_utils_ProcessUtils.h" />
<move
file='${build.dir}/org_voltdb_jni_ExecutionEngine.h'
todir='${src.ee.dir}'
Expand All @@ -777,6 +778,10 @@ NATIVE EE STUFF
file='${build.dir}/org_voltdb_utils_ThreadUtils.h'
todir='${src.ee.dir}'
/>
<move
file='${build.dir}/org_voltdb_utils_ProcessUtils.h'
todir='${src.ee.dir}'
/>
</target>

<target name='uptodate_jni_h.check' depends='jnicompile_temp'>
Expand All @@ -794,13 +799,19 @@ NATIVE EE STUFF
file1="${src.ee.dir}/org_voltdb_utils_ThreadUtils.h"
file2="${build.dir}/org_voltdb_utils_ThreadUtils.h"
/>
<filesmatch
file1="${src.ee.dir}/org_voltdb_utils_ProcessUtils.h"
file2="${build.dir}/org_voltdb_utils_ProcessUtils.h"
/>
</and>
</condition>
</target>

<target name='jnicompile_temp'>
<delete file="${build.dir}/org_voltdb_jni_ExecutionEngine.h"/>
<delete file="${build.dir}/org_voltdb_utils_DBBPool.h" />
<delete file="${build.dir}/org_voltdb_utils_ThreadUtils.h" />
<delete file="${build.dir}/org_voltdb_utils_ProcessUtils.h" />
<javah
classpathref="project.classpath"
force="yes"
Expand All @@ -822,6 +833,13 @@ NATIVE EE STUFF
class="org.voltdb.utils.ThreadUtils"
destdir="${build.dir}"
/>
<javah
classpathref="project.classpath"
force="yes"
verbose="yes"
class="org.voltdb.utils.ProcessUtils"
destdir="${build.dir}"
/>
</target>

<target name="eecheck" depends="ee, eecheck-build"
Expand Down
10 changes: 5 additions & 5 deletions src/ee/indexes/indexkey.h
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ class IntsKey {
* in this next uint64_t.
*/
if (intraKeyOffset < 0) {
intraKeyOffset = sizeof(uint64_t) - 1;
intraKeyOffset = static_cast<int>(sizeof(uint64_t)) - 1;
keyOffset++;
}
}
Expand All @@ -174,7 +174,7 @@ class IntsKey {
retval |= (0xFF & (data[keyOffset] >> (intraKeyOffset * 8))) << (ii * 8);
intraKeyOffset--;
if (intraKeyOffset < 0) {
intraKeyOffset = sizeof(uint64_t) - 1;
intraKeyOffset = static_cast<int>(sizeof(uint64_t)) - 1;
keyOffset++;
}
}
Expand All @@ -184,7 +184,7 @@ class IntsKey {
std::string debug( const voltdb::TupleSchema *keySchema) const {
std::ostringstream buffer;
int keyOffset = 0;
int intraKeyOffset = sizeof(uint64_t) - 1;
int intraKeyOffset = static_cast<int>(sizeof(uint64_t)) - 1;
const int columnCount = keySchema->columnCount();
for (int ii = 0; ii < columnCount; ii++) {
switch(keySchema->columnType(ii)) {
Expand Down Expand Up @@ -222,7 +222,7 @@ class IntsKey {
const TupleSchema *keySchema = tuple->getSchema();
const int columnCount = keySchema->columnCount();
int keyOffset = 0;
int intraKeyOffset = sizeof(uint64_t) - 1;
int intraKeyOffset = static_cast<int>(sizeof(uint64_t)) - 1;
for (int ii = 0; ii < columnCount; ii++) {
switch(keySchema->columnType(ii)) {
case voltdb::VALUE_TYPE_BIGINT: {
Expand Down Expand Up @@ -260,7 +260,7 @@ class IntsKey {
::memset(data, 0, keySize * sizeof(uint64_t));
const int columnCount = keySchema->columnCount();
int keyOffset = 0;
int intraKeyOffset = sizeof(uint64_t) - 1;
int intraKeyOffset = static_cast<int>(sizeof(uint64_t)) - 1;
for (int ii = 0; ii < columnCount; ii++) {
switch(keySchema->columnType(ii)) {
case voltdb::VALUE_TYPE_BIGINT: {
Expand Down
21 changes: 21 additions & 0 deletions src/ee/voltdbjni.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
#include "org_voltdb_jni_ExecutionEngine.h" // the header file output by javah
#include "org_voltdb_utils_DBBPool.h" //Utility method for DBBContainer
#include "org_voltdb_utils_ThreadUtils.h" // the header file output by javah
#include "org_voltdb_utils_ProcessUtils.h" // the header file output by javah

#include "boost/shared_ptr.hpp"
#include "boost/scoped_array.hpp"
Expand Down Expand Up @@ -1273,6 +1274,26 @@ SHAREDLIB_JNIEXPORT jint JNICALL Java_org_voltdb_utils_ThreadUtils_getNumCores
return static_cast<jint>(NUM_PROCS);
}

/*
* Class: org_voltdb_utils_ProcessUtils
* Method: fork
* Signature: ()I
*/
SHAREDLIB_JNIEXPORT jint JNICALL Java_org_voltdb_utils_ProcessUtils_fork
(JNIEnv *env, jclass clazz) {
return (fork());
}

/*
* Class: org_voltdb_utils_ProcessUtils
* Method: kill
* Signature: ()I
*/
SHAREDLIB_JNIEXPORT void JNICALL Java_org_voltdb_utils_ProcessUtils_kill
(JNIEnv *env, jclass clazz, jint pid) {
kill(static_cast<int>(pid), SIGKILL);
}

// ----------------------------------------------------------------------------
// READ/WRITE TRACKING
// ----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions src/frontend/edu/brown/catalog/FixCatalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public static Catalog updateCatalog(Catalog catalog, ClusterConfiguration cc) {

int proc_port = HStoreConstants.DEFAULT_PORT;
int messenger_port = proc_port + HStoreConstants.MESSENGER_PORT_OFFSET;
int jvmsnapshot_port = proc_port + HStoreConstants.JVMSNAPSHOT_PORT_OFFSET;

// Now create the sites for this host
for (Integer siteid : cc.getSites(host)) {
Expand All @@ -113,6 +114,7 @@ public static Catalog updateCatalog(Catalog catalog, ClusterConfiguration cc) {
catalog_site.setHost(catalog_host);
catalog_site.setProc_port(proc_port++);
catalog_site.setMessenger_port(messenger_port++);
catalog_site.setJVMSnapshot_port(jvmsnapshot_port++);

// Add all the partitions
for (Integer partition_id : cc.getPartitionIds(host, siteid)) {
Expand Down
2 changes: 2 additions & 0 deletions src/frontend/edu/brown/hstore/HStoreConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ public abstract class HStoreConstants {

public static final int MESSENGER_PORT_OFFSET = 10000;

public static final int JVMSNAPSHOT_PORT_OFFSET = 20000;

// ----------------------------------------------------------------------------
// THREAD NAMES
// ----------------------------------------------------------------------------
Expand Down
Loading