From 2909d23989f99e75366116c380f38d78a01c73d5 Mon Sep 17 00:00:00 2001 From: Yulong Wu Date: Sat, 7 Mar 2020 18:56:38 +0000 Subject: [PATCH] Introduce API v2 --- .gitignore | 4 +- README.md | 122 +++-- conf/server.json | 26 +- pom.xml | 224 ++++---- .../ranksays/rocksdb/server/ApiRequest.java | 22 + .../ranksays/rocksdb/server/ApiResponse.java | 35 ++ .../com/ranksays/rocksdb/server/Config.java | 78 +++ .../com/ranksays/rocksdb/server/Configs.java | 67 --- .../rocksdb/server/DatabaseManager.java | 119 ++++ .../ranksays/rocksdb/server/HttpHandler.java | 511 +++++------------- .../com/ranksays/rocksdb/server/Main.java | 91 ++-- .../com/ranksays/rocksdb/server/Response.java | 91 ---- src/main/resources/log4j2.xml | 8 +- 13 files changed, 649 insertions(+), 749 deletions(-) create mode 100644 src/main/java/com/ranksays/rocksdb/server/ApiRequest.java create mode 100644 src/main/java/com/ranksays/rocksdb/server/ApiResponse.java create mode 100644 src/main/java/com/ranksays/rocksdb/server/Config.java delete mode 100644 src/main/java/com/ranksays/rocksdb/server/Configs.java create mode 100644 src/main/java/com/ranksays/rocksdb/server/DatabaseManager.java delete mode 100644 src/main/java/com/ranksays/rocksdb/server/Response.java diff --git a/.gitignore b/.gitignore index 261b759..0878837 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ # Maven build target folder target/ dist/ +log/ +data/ # IntelliJ .idea/ @@ -12,4 +14,4 @@ dist/ # Eclipse .classpath .project -.settings/ \ No newline at end of file +.settings/ diff --git a/README.md b/README.md index dd46f68..9a83e27 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,78 @@ -# rocksdb-server -A tiny HTTP-based server for RocksDB +# RocksDB Server + +This is a lightweight RocksDB Server based on jetty. ## Features -* Use RocksDB as key-value storage engine -* HTTP-base API interface -* Authorization -* Support basic operations: get, put, remove, drop_database -* Batch get/put/remove (not atomic) +* Multiple concurrent databases; +* Simple API interface and client libraries; +* Basic Authentication; +* Batch `put`, `get` and `remove` operations. ## Prerequisites -You need to have Java 8+ installed. Currently supported platform includes Mac OS X and Linux. -## How to use +You need to have a Java 8 or above runtime installed. + +## Get started + +1. Download and unzip the latest binary release from [here](https://github.com/iamyulong/rocksdb-server/releases); +2. Update the configuration file at `./conf/server.json`; +3. Start up the server: `./bin/startup.sh`. + +Once boot up, you can check if it's working via http://localhost:8516. + +## Install as system service (Ubuntu) + +1. Copy the following config to `/etc/init.d/rocksdb-server`, after replacing `SERVER_ROOT` with a real path: + + ```bash + #!/bin/bash + + ### BEGIN INIT INFO + # Provides: rocksdb-server + # Required-Start: $network + # Required-Stop: $network + # Default-Start: 2 3 4 5 + # Default-Stop: 0 1 6 + # Short-Description: start/stop rocksdb-server + ### END INIT INFO + + SERVER_ROOT=/path/to/rocksdb/server + + case $1 in + start) + /bin/bash $SERVER_ROOT/bin/startup.sh + ;; + stop) + /bin/bash $SERVER_ROOT/bin/shutdown.sh + ;; + restart) + /bin/bash $SERVER_ROOT/bin/restart.sh + ;; + esac + exit 0 + ``` + +2. Update the permissions and enable service: + + ```bash + chmod 755 /etc/init.d/rocksdb-server + update-rc.d rocksdb-server defaults + ``` + +3. Now, you can start/stop/restart your server via the following commands: + + ```bash + service rocksdb-server start + service rocksdb-server stop + service rocksdb-server restart + ``` -1. Download the latest release from . -2. Unarchive and modify the configuration file at `conf/server.json`. -3. Start the server via terminal: `./bin/startup.sh`. -4. You can verify if it is working by visiting: ## Build from source ```bash git clone https://github.com/iamyulong/rocksdb-server cd rocksdb-server -mvn package -``` - -## Install as system service (Linux) - -Edit SERVER_ROOT in the following script and copy to `/etc/init.d/rocksdb-server`. -```bash -#!/bin/bash - -### BEGIN INIT INFO -# Provides: rocksdb-server -# Required-Start: $network -# Required-Stop: $network -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start/stop rocksdb-server -### END INIT INFO - -SERVER_ROOT=/path/to/rocksdb/server - -case $1 in - start) - /bin/bash $SERVER_ROOT/bin/startup.sh - ;; - stop) - /bin/bash $SERVER_ROOT/bin/shutdown.sh - ;; - restart) - /bin/bash $SERVER_ROOT/bin/restart.sh - ;; -esac -exit 0 -``` -Change its permissions and add the correct symlinks automatically. -```bash -chmod 755 /etc/init.d/rocksdb-server -update-rc.d rocksdb-server defaults -``` -From now on, you can start/stop/restart your server via the following commands: -```bash -service rocksdb-server start -service rocksdb-server stop -service rocksdb-server restart -``` +mvn clean install +``` \ No newline at end of file diff --git a/conf/server.json b/conf/server.json index e3571ec..8a2baad 100644 --- a/conf/server.json +++ b/conf/server.json @@ -1,12 +1,18 @@ { - "listen": "127.0.0.1", - "port": 8516, - - "auth": { - "enabled": true, - "username": "username", - "password": "password" - }, - - "data_dir": "data" + "node": { + "host": "127.0.0.1", + "port": 8516 + }, + "auth": { + "enabled": false, + "username": "admin", + "password": "admin" + }, + "db": { + "maxOpenFiles": 256, + "blockSize": 4096, + "rowCache": 33554432, + "writeBufferSize": 4194304 + }, + "dataDir": "data" } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 070cc22..0894a6d 100644 --- a/pom.xml +++ b/pom.xml @@ -1,127 +1,127 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - com.yulongs - rocksdb-server - 1.1 - jar + com.yulongs + rocksdb-server + 2.0 + jar - rocksdb-server - https://github.com/iamyulong/rocksdb-server + rocksdb-server + https://github.com/iamyulong/rocksdb-server - - - org.eclipse.jetty - jetty-server - 9.4.25.v20191220 - + + + org.eclipse.jetty + jetty-server + 9.4.27.v20200227 + - - org.rocksdb - rocksdbjni - 6.5.2 - + + org.rocksdb + rocksdbjni + 6.6.4 + - - org.json - json - 20190722 - + + com.fasterxml.jackson.core + jackson-databind + 2.10.3 + - - junit - junit - 4.13 - test - + + junit + junit + 4.13 + test + - - org.apache.logging.log4j - log4j-core - 2.13.0 - - + + org.apache.logging.log4j + log4j-core + 2.13.1 + + - - UTF-8 - ${project.basedir}/dist/rocksdb-server-${project.version} - + + UTF-8 + ${project.basedir}/dist/rocksdb-server-${project.version} + - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - package - - copy-dependencies - - - ${project.dist.dir}/lib - - - - + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + package + + copy-dependencies + + + ${project.dist.dir}/lib + + + + - - org.apache.maven.plugins - maven-jar-plugin - 3.0.0 - - ${project.dist.dir}/lib - - + + org.apache.maven.plugins + maven-jar-plugin + 3.0.0 + + ${project.dist.dir}/lib + + - - maven-antrun-plugin - 1.7 - - - package - - - - - - - - - - - - - run - - - - + + maven-antrun-plugin + 1.7 + + + package + + + + + + + + + + + + + run + + + + - - maven-clean-plugin - 3.1.0 - - - - dist - false - - - - - - + + maven-clean-plugin + 3.1.0 + + + + dist + false + + + + + + diff --git a/src/main/java/com/ranksays/rocksdb/server/ApiRequest.java b/src/main/java/com/ranksays/rocksdb/server/ApiRequest.java new file mode 100644 index 0000000..337e8dd --- /dev/null +++ b/src/main/java/com/ranksays/rocksdb/server/ApiRequest.java @@ -0,0 +1,22 @@ +package com.ranksays.rocksdb.server; + +import java.util.ArrayList; +import java.util.List; + +public class ApiRequest { + private String name; + private List keys = new ArrayList<>(); + private List values = new ArrayList<>(); + + public String getName() { + return name; + } + + public List getKeys() { + return keys; + } + + public List getValues() { + return values; + } +} diff --git a/src/main/java/com/ranksays/rocksdb/server/ApiResponse.java b/src/main/java/com/ranksays/rocksdb/server/ApiResponse.java new file mode 100644 index 0000000..8b592a8 --- /dev/null +++ b/src/main/java/com/ranksays/rocksdb/server/ApiResponse.java @@ -0,0 +1,35 @@ +package com.ranksays.rocksdb.server; + + +public class ApiResponse { + + private Integer code; + private String message; + private Object body; + + public ApiResponse() { + + } + + public ApiResponse(Integer code, String message) { + this(code, message, null); + } + + public ApiResponse(Integer code, String message, Object body) { + this.code = code; + this.message = message; + this.body = body; + } + + public Integer getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public Object getBody() { + return body; + } +} diff --git a/src/main/java/com/ranksays/rocksdb/server/Config.java b/src/main/java/com/ranksays/rocksdb/server/Config.java new file mode 100644 index 0000000..5538f59 --- /dev/null +++ b/src/main/java/com/ranksays/rocksdb/server/Config.java @@ -0,0 +1,78 @@ +package com.ranksays.rocksdb.server; + +public class Config { + private Node node; + private Auth auth; + private Db db; + private String dataDir; + + public Node getNode() { + return node; + } + + public Auth getAuth() { + return auth; + } + + public Db getDb() { + return db; + } + + public String getDataDir() { + return dataDir; + } + + public class Node { + private String host; + private Integer port; + + public String getHost() { + return host; + } + + public Integer getPort() { + return port; + } + } + + public class Auth { + private boolean enabled; + private String username; + private String password; + + public boolean isEnabled() { + return enabled; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + } + + public class Db { + private Integer maxOpenFiles; + private Integer blockSize; + private Integer rowCache; + private Integer writeBufferSize; + + public Integer getMaxOpenFiles() { + return maxOpenFiles; + } + + public Integer getBlockSize() { + return blockSize; + } + + public Integer getRowCache() { + return rowCache; + } + + public Integer getWriteBufferSize() { + return writeBufferSize; + } + } +} diff --git a/src/main/java/com/ranksays/rocksdb/server/Configs.java b/src/main/java/com/ranksays/rocksdb/server/Configs.java deleted file mode 100644 index 4f0a374..0000000 --- a/src/main/java/com/ranksays/rocksdb/server/Configs.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.ranksays.rocksdb.server; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.JSONException; -import org.json.JSONObject; - -/** - * Global configurations - * - */ -public class Configs { - private static final Logger logger = LogManager.getLogger(Configs.class); - - private static final String SERVER_CONFIG_FILE = "conf/server.json"; - - public static String listen = "127.0.0.1"; - public static int port = 8080; - - public static boolean authEnabled = false; - public static String username = null; - public static String password = null; - - public static String dataDir = "data"; - - public static String readFile(String path, String encoding) throws IOException { - byte[] encoded = Files.readAllBytes(Paths.get(path)); - - return new String(encoded, encoding); - } - - public static boolean load() { - try { - String configs = readFile(SERVER_CONFIG_FILE, "UTF-8"); - JSONObject obj = new JSONObject(configs); - - if (obj.has("listen")) { - listen = obj.getString("listen"); - } - if (obj.has("port")) { - port = obj.getInt("port"); - } - - if (obj.has("auth")) { - JSONObject o = obj.getJSONObject("auth"); - authEnabled = o.getBoolean("enabled"); - if (authEnabled) { - username = o.getString("username"); - password = o.getString("password"); - } - } - - if (obj.has("data_dir")) { - dataDir = obj.getString("data_dir"); - } - } catch (IOException | JSONException e) { - logger.error("Failed to parse " + SERVER_CONFIG_FILE + ": " + e.getMessage()); - return false; - } - - return true; - } -} diff --git a/src/main/java/com/ranksays/rocksdb/server/DatabaseManager.java b/src/main/java/com/ranksays/rocksdb/server/DatabaseManager.java new file mode 100644 index 0000000..f4d6b53 --- /dev/null +++ b/src/main/java/com/ranksays/rocksdb/server/DatabaseManager.java @@ -0,0 +1,119 @@ +package com.ranksays.rocksdb.server; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.rocksdb.LRUCache; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +public class DatabaseManager implements Closeable { + + private static final Logger logger = LogManager.getLogger(Main.class); + + private AtomicBoolean closed = new AtomicBoolean(false); + private Config config; + private Map databases; + + public DatabaseManager(Config config) { + this.config = config; + this.databases = new HashMap<>(); + } + + /** + * Creates a {@link RocksDB} instance for the given database. + * + * @param name the name of the database + * @return a database instance + * @implNote created instance may be cached and shared among clients. + */ + public synchronized RocksDB open(String name) { + if (!databases.containsKey(name) && !closed.get()) { + Options opts = new Options(); + opts.setCreateIfMissing(true); + if (config.getDb().getBlockSize() != null) { + opts.setArenaBlockSize(config.getDb().getBlockSize()); + } + if (config.getDb().getRowCache() != null) { + opts.setRowCache(new LRUCache(config.getDb().getRowCache())); + } + if (config.getDb().getMaxOpenFiles() != null) { + opts.setMaxOpenFiles(config.getDb().getMaxOpenFiles()); + } + if (config.getDb().getWriteBufferSize() != null) { + opts.setWriteBufferSize(config.getDb().getWriteBufferSize()); + } + + RocksDB db; + try { + db = RocksDB.open(opts, config.getDataDir() + "/" + name); + databases.put(name, db); + return db; + } catch (RocksDBException e) { + logger.error("Failed to open database: name = {}", name, e); + } + } + + return databases.get(name); + } + + /** + * Closes the {@link RocksDB} instance for the given database, if already opened. + *

+ * All resources retained by the database will be released afterwards. + * + * @param name the database name + */ + public synchronized void close(String name) { + RocksDB db = databases.remove(name); + if (db != null) { + db.close(); + } + } + + /** + * Deletes a database if exists. + * + * @param name the database name. + */ + public synchronized void drop(String name) throws IOException { + close(name); + deleteFileRecursively(new File(config.getDataDir(), name)); + } + + /** + * Closes all open {@link RocksDB} instances. + * + * @throws IOException when any IO error occurs + */ + @Override + public synchronized void close() { + List names = new ArrayList<>(databases.keySet()); + for (String name : names) { + close(name); + } + closed.set(true); + } + + private void deleteFileRecursively(File f) throws IOException { + if (f.exists()) { + if (f.isDirectory()) { + for (File c : f.listFiles()) { + deleteFileRecursively(c); + } + } + if (!f.delete()) { + throw new IOException("Failed to delete file: " + f); + } + } + } +} diff --git a/src/main/java/com/ranksays/rocksdb/server/HttpHandler.java b/src/main/java/com/ranksays/rocksdb/server/HttpHandler.java index 2e30db8..39cab7d 100644 --- a/src/main/java/com/ranksays/rocksdb/server/HttpHandler.java +++ b/src/main/java/com/ranksays/rocksdb/server/HttpHandler.java @@ -1,450 +1,239 @@ package com.ranksays.rocksdb.server; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.util.Base64; -import java.util.Map; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; -import org.json.JSONArray; -import org.json.JSONObject; -import org.rocksdb.Options; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -/** - * HTTP requests handler - * - */ +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + public class HttpHandler extends AbstractHandler { private static final Logger logger = LogManager.getLogger(HttpHandler.class); - /** - * Default encoding - */ - protected static final String ENCODING = "UTF-8"; + private static final String encoding = "UTF-8"; + private static final ObjectMapper mapper = new ObjectMapper(); - /** - * Opened databases - */ - protected Map openDBs; + protected Config config; + protected DatabaseManager manager; - /*** - * Construct a HttpHandler. - * - * @param databases - * opened databases - */ - public HttpHandler(Map databases) { - super(); - this.openDBs = databases; + public HttpHandler(Config config, DatabaseManager manager) { + this.config = config; + this.manager = manager; } - /** - * HTTP requests processor, authenticating, parsing and dispatching. - * - * @param target - * target URI - * @param baseRequest - * base request - * @param request - * HTTP servlet request - * @param response - * HTTP servlet response - * - * @throws IOException - * @throws ServletException - */ + @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - Response resp = null; - try { - // parse request - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - BufferedInputStream in = new BufferedInputStream(request.getInputStream()); - for (int c; (c = in.read()) != -1;) { - buf.write(c); + // check basic auth + if (!checkAuth(request)) { + logger.warn("Unauthorized access: ip = {}", request.getRemoteAddr()); + response.setHeader("WWW-Authenticate", "Basic realm=\"RocksDB Server\""); + writeResponse(response, new ApiResponse(401, "Unauthorized")); + return; } - JSONObject req = new JSONObject(buf.size() > 0 ? buf.toString(ENCODING) : "{}"); - // authorization (Basic Auth prioritizes) - String basicAuth = request.getHeader("Authorization"); - if (basicAuth != null && basicAuth.startsWith("Basic ")) { - req.put("auth", basicAuth.substring(6)); + // read input + ApiRequest apiRequest; + try { + apiRequest = mapper.readValue(request.getInputStream(), ApiRequest.class); + } catch (IOException e) { + logger.error("Received invalid request from client", e); + writeResponse(response, new ApiResponse(400, "Bad request")); + return; } - if ("/get".equals(target)) { - resp = doGet(req); - } else if ("/put".equals(target)) { - resp = doPut(req); - } else if ("/remove".equals(target)) { - resp = doRemove(req); - } else if ("/drop_database".equals(target)) { - resp = doDropDatabase(req); - } else if ("/".equals(target)) { - resp = new Response("This rocksdb-server is working"); - } else { - resp = new Response(404, "Sorry, that page does not exist"); + try { + ApiResponse apiResponse; + switch (target) { + case "/get": + apiResponse = doGet(apiRequest); + break; + case "/put": + apiResponse = doPut(apiRequest); + break; + case "/delete": + apiResponse = doDelete(apiRequest); + break; + case "/create": + apiResponse = doCreate(apiRequest); + break; + case "/drop": + apiResponse = doDrop(apiRequest); + break; + case "/stats": + apiResponse = doStats(apiRequest); + break; + default: + writeResponse(response, new ApiResponse(404, "Not found")); + return; + } + writeResponse(response, apiResponse); + } catch (Exception e) { + logger.error("Internal error: request = {}", apiRequest, e); + writeResponse(response, new ApiResponse(500, "Internal error: " + e.getMessage())); } - } catch (Exception e) { - resp = new Response(500, "Internal server error"); - logger.error("Internal error: " + e.getMessage()); + } finally { + baseRequest.setHandled(true); } - - response.setContentType("application/json; charset=utf-8"); - response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().println(resp.toJSON()); - - baseRequest.setHandled(true); } /** - * Get operation handler. - * - * @param req - * request JSONObject - * @return API response - * @throws IOException - * @throws ServletException + * Check if the request has bee authorized. */ - protected Response doGet(JSONObject req) throws IOException, ServletException { - Response resp = new Response(); - - // parse parameters & open database - String db = null; - byte[][] keys = null; - RocksDB rdb = null; - if (!authorize(req, resp) || (db = parseDB(req, resp)) == null || (keys = parseKeys(req, resp)) == null - || (rdb = openDatabase(db, resp)) == null) { - return resp; + protected boolean checkAuth(HttpServletRequest request) { + if (!config.getAuth().isEnabled()) { + return true; } try { - byte[][] values = new byte[keys.length][]; - for (int i = 0; i < keys.length; i++) { - values[i] = rdb.get(keys[i]); - } - - JSONArray arr = new JSONArray(); - for (int i = 0; i < keys.length; i++) { - if (values[i] == null) { - arr.put(JSONObject.NULL); - } else { - arr.put(Base64.getEncoder().encodeToString(values[i])); + String authorization = request.getHeader("Authorization"); + if (authorization != null && authorization.startsWith("Basic ")) { + byte[] decoded = Base64.getDecoder().decode(authorization.substring(6)); + String[] tokens = new String(decoded, encoding).split(":"); + + if (config.getAuth().getUsername().equals(tokens[0]) + && config.getAuth().getPassword().equals(tokens[1])) { + return true; } } - resp.setResults(arr); - } catch (RocksDBException e) { - resp.setCode(Response.CODE_INTERNAL_ERROR); - resp.setMessage("Internal server error"); - - logger.error("get operation failed: " + e.getMessage()); + } catch (Exception e) { + // do nothing } - return resp; + return false; } /** - * Put operation handler. - * - * @param req - * request JSONObject - * @return API response + * Writes an API response. + * + * @param response the HTTP response + * @param apiResponse the API response * @throws IOException - * @throws ServletException */ - protected Response doPut(JSONObject req) throws IOException, ServletException { - Response resp = new Response(); - - // parse parameters & open database - String db = null; - byte[][] keys = null, values = null; - RocksDB rdb = null; - if (!authorize(req, resp) || (db = parseDB(req, resp)) == null || (keys = parseKeys(req, resp)) == null - || (values = parseValues(req, resp)) == null || (rdb = openDatabase(db, resp)) == null) { - return resp; - } - - if (keys.length != values.length) { - resp.setCode(Response.CODE_KEY_VALUE_MISMATCH); - resp.setMessage("Number of key and value does not match"); - return resp; - } - - try { - for (int i = 0; i < keys.length; i++) { - rdb.put(keys[i], values[i]); - } - } catch (RocksDBException e) { - resp.setCode(Response.CODE_INTERNAL_ERROR); - resp.setMessage("Internal server error"); - - logger.error("put operation failed: " + e.getMessage()); - } - - return resp; + protected void writeResponse(HttpServletResponse response, ApiResponse apiResponse) throws IOException { + response.setContentType("application/json; charset=utf-8"); + response.setStatus(apiResponse.getCode()); + mapper.writeValue(response.getOutputStream(), apiResponse); } /** - * remove operation handler. - * - * @param req - * request JSONObject - * @return API response - * @throws IOException - * @throws ServletException + * handles a get request. */ - protected Response doRemove(JSONObject req) throws IOException, ServletException { - Response resp = new Response(); - - // parse parameters & open database - String db = null; - byte[][] keys = null; - RocksDB rdb = null; - if (!authorize(req, resp) || (db = parseDB(req, resp)) == null || (keys = parseKeys(req, resp)) == null - || (rdb = openDatabase(db, resp)) == null) { - return resp; - } - - try { - for (int i = 0; i < keys.length; i++) { - rdb.remove(keys[i]); - } - } catch (RocksDBException e) { - resp.setCode(Response.CODE_INTERNAL_ERROR); - resp.setMessage("Internal server error"); + protected ApiResponse doGet(ApiRequest request) throws RocksDBException { + String name = request.getName(); + List keys = decodeBase64(request.getKeys()); - logger.error("remove operation failed: " + e.getMessage()); + List values = new ArrayList<>(); + RocksDB db = manager.open(name); + for (byte[] key : keys) { + values.add(db.get(key)); } - return resp; + return new ApiResponse(200, "OK", encodeBase64(values)); } /** - * Drop database operation handler. - * - * @param req - * request JSONObject - * @return API response - * @throws IOException - * @throws ServletException + * handles a put request. */ - protected Response doDropDatabase(JSONObject req) throws IOException, ServletException { - Response resp = new Response(); - - // parse parameters - String db = null; - if (!authorize(req, resp) || (db = parseDB(req, resp)) == null) { - return resp; - } - - synchronized (openDBs) { - if (openDBs.containsKey(db)) { - openDBs.get(db).close(); - openDBs.remove(db); - } - - try { - File f = new File(Configs.dataDir, db); - deleteFile(f); - } catch (IOException e) { - resp.setCode(Response.CODE_INTERNAL_ERROR); - resp.setMessage("Internal server error"); - - logger.error("dropping database failed: " + e.getMessage()); + protected ApiResponse doPut(ApiRequest request) throws RocksDBException { + String name = request.getName(); + List keys = decodeBase64(request.getKeys()); + List values = decodeBase64(request.getValues()); + if (keys.size() != values.size()) { + throw new IllegalArgumentException("# of keys and values don't match"); + } + + RocksDB db = manager.open(name); + for (int i = 0; i < keys.size(); i++) { + if (values.get(i) == null) { + db.delete(keys.get(i)); + } else { + db.put(keys.get(i), values.get(i)); } } - return resp; + return new ApiResponse(200, "OK"); } /** - * Basic authorization - * - * @param req - * HTTP request header - * @return true if authorized otherwise false. + * handles a delete request. */ - private boolean authorize(JSONObject req, Response resp) { - if (Configs.authEnabled) { - if (!req.isNull("auth")) { - try { - byte[] decoded = Base64.getDecoder().decode(req.getString("auth")); - String[] tokens = new String(decoded, ENCODING).split(":"); - - if (Configs.username.equals(tokens[0]) && Configs.password.equals(tokens[1])) { - return true; - } - } catch (Exception e) { - // do nothing - } - } - } else { - return true; + protected ApiResponse doDelete(ApiRequest request) throws RocksDBException { + String name = request.getName(); + List keys = decodeBase64(request.getKeys()); + + RocksDB db = manager.open(name); + for (byte[] key : keys) { + db.delete(key); } - resp.setCode(Response.CODE_UNAUTHORIZED); - resp.setMessage("Not authorized"); - return false; + return new ApiResponse(200, "OK"); } /** - * Parse database name from the request. - * - * @param req - * request body - * @param resp - * response container - * @return a valid DB name; or null if the 'db' key does not exist or the - * corresponding value is JSONObject.NULL or does not match the - * specifications. + * handles a create request. */ - private String parseDB(JSONObject req, Response resp) { + protected ApiResponse doCreate(ApiRequest request) throws RocksDBException { + String name = request.getName(); - if (!req.isNull("db")) { - String db = req.getString("db"); - - if (db.matches("[-_.A-Za-z0-9]+")) { - return db; - } - } + manager.open(name); - resp.setCode(Response.CODE_INVALID_DB_NAME); - resp.setMessage("Invalid database name"); - return null; + return new ApiResponse(200, "OK"); } /** - * parse key(s) from the request. - * - * @param req - * request body - * @param resp - * response container - * @return a byte array; or null if the 'keys' key does not exist or the - * corresponding value is JSONObject.NULL or is not a JSONArray of - * Base64-encoded strings. + * handles a drop request. */ - private byte[][] parseKeys(JSONObject req, Response resp) { + protected ApiResponse doDrop(ApiRequest request) throws RocksDBException, IOException { + String name = request.getName(); - if (!req.isNull("keys")) { - try { - JSONArray arr = req.getJSONArray("keys"); - - byte[][] result = new byte[arr.length()][]; - for (int i = 0; i < arr.length(); i++) { - result[i] = Base64.getDecoder().decode(arr.getString(i)); - } - return result; - } catch (Exception e) { - // do nothing - } - } + manager.drop(name); - resp.setCode(Response.CODE_INVALID_KEY); - resp.setMessage("Invalid key(s)"); - return null; + return new ApiResponse(200, "OK"); } /** - * parse value(s) from the request. - * - * @param req - * request body - * @param resp - * response container - * @return a byte array; or null if the 'values' key does not exist or the - * corresponding value is JSONObject.NULL or is not a JSONArray of - * Base64-encoded strings. + * handles a stats request. */ - private byte[][] parseValues(JSONObject req, Response resp) { + protected ApiResponse doStats(ApiRequest request) throws RocksDBException, IOException { + String name = request.getName(); - if (!req.isNull("values")) { - try { - JSONArray arr = req.getJSONArray("values"); - - byte[][] result = new byte[arr.length()][]; - for (int i = 0; i < arr.length(); i++) { - result[i] = Base64.getDecoder().decode(arr.getString(i)); - } - return result; - } catch (Exception e) { - // do nothing - } - } + RocksDB db = manager.open(name); + String stats = db.getProperty("rocksdb.stats"); - resp.setCode(Response.CODE_INVALID_VALUE); - resp.setMessage("Invalid value(s)"); - return null; + return new ApiResponse(200, "OK", stats); } - /** - * Open a specified database (create if missing). - * - * @param db - * the DB name - * @param resp - * response container - * @return the RocksDB reference or null if failed. - */ - private RocksDB openDatabase(String db, Response resp) { - synchronized (openDBs) { - if (openDBs.containsKey(db)) { - return openDBs.get(db); - } else { - Options opts = new Options(); - opts.setCreateIfMissing(true); - - RocksDB rdb = null; - try { - rdb = RocksDB.open(opts, Configs.dataDir + "/" + db); - openDBs.put(db, rdb); - return rdb; - } catch (RocksDBException e) { - String msg = "Failed to open database: " + e.getMessage(); - logger.error(msg); - - resp.setCode(Response.CODE_FAILED_TO_OPEN_DB); - resp.setMessage(msg); - } - // If user doesn't call options dispose explicitly, then - // this options instance will be GC'd automatically. - // opts.close(); - } + protected List decodeBase64(List src) { + if (src == null) { + return Collections.emptyList(); } - return null; + return src.stream() + .map(s -> s == null ? null : Base64.getDecoder().decode(s)) + .collect(Collectors.toList()); } - /** - * Delete file or directory recursively. - * - * @param f - * file or directory to delete - * @throws IOException - * failed to delete a file - */ - private void deleteFile(File f) throws IOException { - if (f.exists()) { - if (f.isDirectory()) { - for (File c : f.listFiles()) { - deleteFile(c); - } - } - if (!f.delete()) { - throw new IOException("Failed to delete file: " + f); - } + protected List encodeBase64(List src) { + if (src == null) { + return Collections.emptyList(); } + + return src.stream() + .map(s -> s == null ? null : Base64.getEncoder().encodeToString(s)) + .collect(Collectors.toList()); } } diff --git a/src/main/java/com/ranksays/rocksdb/server/Main.java b/src/main/java/com/ranksays/rocksdb/server/Main.java index fb44386..c309227 100644 --- a/src/main/java/com/ranksays/rocksdb/server/Main.java +++ b/src/main/java/com/ranksays/rocksdb/server/Main.java @@ -1,62 +1,61 @@ package com.ranksays.rocksdb.server; -import java.io.File; -import java.net.BindException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; - +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.eclipse.jetty.server.Server; import org.rocksdb.RocksDB; -/** - * Launcher for RocksDB Server - * - */ +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; + public class Main { private static final Logger logger = LogManager.getLogger(Main.class); - private static Map databases = new HashMap<>(); - - public static void main(String[] args) throws Exception { - - if (Configs.load()) { - // Load RocksDB static libarary - RocksDB.loadLibrary(); - - // Create data folder if missing - new File(Configs.dataDir).mkdirs(); - - // Register shutdown hook - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - synchronized (databases) { - for (RocksDB db : databases.values()) { - if (db != null) { - db.close(); - } - } - logger.info("Server shut down."); - } - } - }); - - // Create Server - InetSocketAddress addr = new InetSocketAddress(Configs.listen, Configs.port); - Server server = new Server(addr); - server.setHandler(new HttpHandler(databases)); + private static ObjectMapper mapper = new ObjectMapper(); + private static final String configFile = "conf/server.json"; + + public static void main(String[] args) { + // Load config file + Config config = null; + try { + config = mapper.readValue(new File(configFile), Config.class); + } catch (IOException e) { + logger.error("Failed to load config file: {}", configFile, e); + System.exit(1); + } + // Load RocksDB static library + RocksDB.loadLibrary(); + + // Create data folder and database manager + new File(config.getDataDir()).mkdirs(); + DatabaseManager manager = new DatabaseManager(config); + + // Launch a web server + InetSocketAddress address = new InetSocketAddress(config.getNode().getHost(), config.getNode().getPort()); + Server server = new Server(address); + server.setHandler(new HttpHandler(config, manager)); + try { + server.start(); + logger.info("RocksDB Server started, visit http://{}:{}", config.getNode().getHost(), config.getNode().getPort()); + server.join(); + } catch (Exception e) { + logger.error("Failed to start RocksDB Server", e); + System.exit(2); + } + + // Register shutdown hook + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + manager.close(); try { - server.start(); - logger.info("Server started."); + server.stop(); server.join(); - } catch (BindException e) { - logger.error("Failed to bind at " + addr + ": " + e.getMessage()); - System.exit(-1); + logger.info("RocksDB Server stopped."); + } catch (Exception e) { + logger.error("Failed to stop RocksDB Server", e); } - } + })); } } diff --git a/src/main/java/com/ranksays/rocksdb/server/Response.java b/src/main/java/com/ranksays/rocksdb/server/Response.java deleted file mode 100644 index c8af1f4..0000000 --- a/src/main/java/com/ranksays/rocksdb/server/Response.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.ranksays.rocksdb.server; - -import org.json.JSONObject; - -/** - * API response protocol. - * - */ -public class Response { - public static final int CODE_OK = 0; - public static final int CODE_UNAUTHORIZED = 401; - public static final int CODE_INTERNAL_ERROR = 500; - public static final int CODE_INVALID_DB_NAME = 1001; - public static final int CODE_INVALID_KEY = 1002; - public static final int CODE_INVALID_VALUE = 1003; - public static final int CODE_FAILED_TO_OPEN_DB = 1004; - public static final int CODE_KEY_VALUE_MISMATCH = 1005; - - public int code; - public String message; - public Object results; - - public Response(int code, String message, Object results) { - super(); - this.code = code; - this.message = message; - this.results = results; - } - - public Response(int code, String message) { - this(code, message, null); - } - - public Response(Object result) { - this(CODE_OK, "Success", result); - } - - public Response() { - this(CODE_OK, "Success", null); - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public Object getResults() { - return results; - } - - public void setResults(Object results) { - this.results = results; - } - - @Override - public String toString() { - return "Response [code=" + code + ", message=" + message + ", results=" + results + "]"; - } - - public JSONObject toJSON() { - JSONObject obj = new JSONObject(); - obj.put("code", code); - obj.put("message", message); - obj.put("results", results); - - return obj; - } - - public static Response fromJSON(JSONObject obj) { - Response resp = new Response(); - - resp.setCode(obj.getInt("code")); - resp.setMessage(obj.getString("message")); - if (!obj.isNull("results")) { - resp.setResults(obj.get("results")); - } - - return resp; - } -} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index e6271b8..b569066 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -3,16 +3,16 @@ - + - + - - + + \ No newline at end of file