diff --git a/ycsb-mongodb/bin/bindings.properties b/ycsb-mongodb/bin/bindings.properties index caf8984..2910f11 100644 --- a/ycsb-mongodb/bin/bindings.properties +++ b/ycsb-mongodb/bin/bindings.properties @@ -25,5 +25,6 @@ # - if the directory contains multiple versions with different classes, # use a dash with the version. (e.g. cassandra-7, cassandra-cql) # +jdbc:site.ycsb.db.JdbcDBClient mongodb:site.ycsb.db.MongoDbClient mongodb-async:site.ycsb.db.AsyncMongoDbClient \ No newline at end of file diff --git a/ycsb-mongodb/bin/ycsb b/ycsb-mongodb/bin/ycsb index 7f2c916..a29021b 100755 --- a/ycsb-mongodb/bin/ycsb +++ b/ycsb-mongodb/bin/ycsb @@ -45,6 +45,7 @@ COMMANDS = { } DATABASES = { + "jdbc" : "site.ycsb.db.JdbcDBClient", "mongodb" : "site.ycsb.db.MongoDbClient", "mongodb-async": "site.ycsb.db.AsyncMongoDbClient", } diff --git a/ycsb-mongodb/jdbc/README.md b/ycsb-mongodb/jdbc/README.md new file mode 100644 index 0000000..97b0543 --- /dev/null +++ b/ycsb-mongodb/jdbc/README.md @@ -0,0 +1,135 @@ + + +# JDBC Driver for YCSB +This driver enables YCSB to work with databases accessible via the JDBC protocol. + +## Getting Started +### 1. Start your database +This driver will connect to databases that use the JDBC protocol, please refer to your databases documentation on information on how to install, configure and start your system. + +### 2. Set up YCSB +You can clone the YCSB project and compile it to stay up to date with the latest changes. Or you can just download the latest release and unpack it. Either way, instructions for doing so can be found here: https://github.com/brianfrankcooper/YCSB. + +### 3. Configure your database and table. +You can name your database what ever you want, you will need to provide the database name in the JDBC connection string. + +You can name your table whatever you like also, but it needs to be specified using the YCSB core properties, the default is to just use 'usertable' as the table name. + +The expected table schema will look similar to the following, syntactical differences may exist with your specific database: + +```sql +CREATE TABLE usertable ( + YCSB_KEY VARCHAR(255) PRIMARY KEY, + FIELD0 TEXT, FIELD1 TEXT, + FIELD2 TEXT, FIELD3 TEXT, + FIELD4 TEXT, FIELD5 TEXT, + FIELD6 TEXT, FIELD7 TEXT, + FIELD8 TEXT, FIELD9 TEXT +); +``` + +Key take aways: + +* The primary key field needs to be named YCSB_KEY +* The other fields need to be prefixed with FIELD and count up starting from 1 +* Add the same number of FIELDs as you specify in the YCSB core properties, default is 10. +* The type of the fields is not so important as long as they can accept strings of the length that you specify in the YCSB core properties, default is 100. + +#### JdbcDBCreateTable Utility +YCSB has a utility to help create your SQL table. NOTE: It does not support all databases flavors, if it does not work for you, you will have to create your table manually with the schema given above. An example usage of the utility: + +```sh +java -cp YCSB_HOME/jdbc-binding/lib/jdbc-binding-0.4.0.jar:mysql-connector-java-5.1.37-bin.jar site.ycsb.db.JdbcDBCreateTable -P db.properties -n usertable +``` + +Hint: you need to include your Driver jar in the classpath as well as specify JDBC connection information via a properties file, and a table name with ```-n```. + +Simply executing the JdbcDBCreateTable class without any other parameters will print out usage information. + +### 4. Configure YCSB connection properties +You need to set the following connection configurations: + +```sh +db.driver=com.mysql.jdbc.Driver +db.url=jdbc:mysql://127.0.0.1:3306/ycsb +db.user=admin +db.passwd=admin +``` + +Be sure to use your driver class, a valid JDBC connection string, and credentials to your database. + +For connection fail-over in a DBMS cluster specify the connection string as follows (example based on Postgres): + +```sh +db.driver=org.postgresql.Driver +db.url=jdbc:postgresql://IP1:PORT1,IP2:PORT2,IP3:PORT3/ycsb +db.user=admin +db.passwd=admin +``` + +For using multiple shards in a DBMS cluster specify the connection string as follows by using `;`as delimiter (example based on PostgreSQL): + +```sh +db.driver=org.postgresql.Driver +db.url=jdbc:postgresql://host1:port1/ycsb;jdbc:postgresql://host2:port2/ycsb +db.user=admin +db.passwd=admin +``` + +You can add these to your workload configuration or a separate properties file and specify it with ```-P``` or you can add the properties individually to your ycsb command with ```-p```. + +### 5. Add your JDBC Driver to the classpath +There are several ways to do this, but a couple easy methods are to put a copy of your Driver jar in ```YCSB_HOME/jdbc-binding/lib/``` or just specify the path to your Driver jar with ```-cp``` in your ycsb command. + +### 6. Running a workload +Before you can actually run the workload, you need to "load" the data first. + +```sh +bin/ycsb load jdbc -P workloads/workloada -P db.properties -cp mysql-connector-java.jar +``` + +Then, you can run the workload: + +```sh +bin/ycsb run jdbc -P workloads/workloada -P db.properties -cp mysql-connector-java.jar +``` + +## Configuration Properties + +```sh +db.driver=com.mysql.jdbc.Driver # The JDBC driver class to use. +db.url=jdbc:mysql://127.0.0.1:3306/ycsb # The Database connection URL. +db.user=admin # User name for the connection. +db.passwd=admin # Password for the connection. +db.batchsize=1000 # The batch size for doing batched inserts. Defaults to 0. Set to >0 to use batching. +jdbc.fetchsize=10 # The JDBC fetch size hinted to the driver. +jdbc.autocommit=true # The JDBC connection auto-commit property for the driver. +jdbc.batchupdateapi=false # Use addBatch()/executeBatch() JDBC methods instead of executeUpdate() for writes (default: false) +db.batchsize=1000 # The number of rows to be batched before commit (or executeBatch() when jdbc.batchupdateapi=true) +``` + +Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for all other YCSB core properties. + +## JDBC Parameter to Improve Insert Performance + +Some JDBC drivers support re-writing batched insert statements into multi-row insert statements. This technique can yield order of magnitude improvement in insert statement performance. To enable this feature: +- **db.batchsize** must be greater than 0. The magniute of the improvement can be adjusted by varying **batchsize**. Start with a small number and increase at small increments until diminishing return in the improvement is observed. +- set **jdbc.batchupdateapi=true** to enable batching. +- set JDBC driver specific connection parameter in **db.url** to enable the rewrite as shown in the examples below: + * MySQL [rewriteBatchedStatements=true](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-configuration-properties.html) with `db.url=jdbc:mysql://127.0.0.1:3306/ycsb?rewriteBatchedStatements=true` + * Postgres [reWriteBatchedInserts=true](https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters) with `db.url=jdbc:postgresql://127.0.0.1:5432/ycsb?reWriteBatchedInserts=true` diff --git a/ycsb-mongodb/jdbc/pom.xml b/ycsb-mongodb/jdbc/pom.xml new file mode 100644 index 0000000..b0520d4 --- /dev/null +++ b/ycsb-mongodb/jdbc/pom.xml @@ -0,0 +1,93 @@ + + + + + 4.0.0 + + site.ycsb + root + 0.18.0-SNAPSHOT + + + + jdbc-binding + JDBC DB Binding + jar + + + + org.apache.openjpa + openjpa-jdbc + ${openjpa.jdbc.version} + + + org.postgresql + postgresql + 42.7.4 + + + site.ycsb + core + ${project.version} + provided + + + junit + junit + 4.12 + test + + + org.hsqldb + hsqldb + 2.3.3 + test + + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven.assembly.version} + + + jar-with-dependencies + + false + + + + package + + single + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + diff --git a/ycsb-mongodb/jdbc/src/main/conf/db.properties b/ycsb-mongodb/jdbc/src/main/conf/db.properties new file mode 100644 index 0000000..e12cc1e --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/conf/db.properties @@ -0,0 +1,22 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Properties file that contains database connection information. + +db.driver=org.h2.Driver +# jdbc.fetchsize=20 +db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb +db.user=sa +db.passwd= diff --git a/ycsb-mongodb/jdbc/src/main/conf/h2.properties b/ycsb-mongodb/jdbc/src/main/conf/h2.properties new file mode 100644 index 0000000..cfde14c --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/conf/h2.properties @@ -0,0 +1,21 @@ +# Copyright (c) 2012 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Properties file that contains database connection information. + +db.driver=org.h2.Driver +db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb +db.user=sa +db.passwd= diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCli.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCli.java new file mode 100644 index 0000000..33b9be1 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCli.java @@ -0,0 +1,181 @@ +/** + * Copyright (c) 2010 - 2016 Yahoo! Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import java.io.FileInputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Enumeration; +import java.util.Properties; + +/** + * Execute a JDBC command line. + * + * @author sudipto + */ +public final class JdbcDBCli { + + private static void usageMessage() { + System.out.println("JdbcCli. Options:"); + System.out.println(" -p key=value properties defined."); + System.out.println(" -P location of the properties file to load."); + System.out.println(" -c SQL command to execute."); + } + + private static void executeCommand(Properties props, String sql) throws SQLException { + String driver = props.getProperty(JdbcDBClient.DRIVER_CLASS); + String username = props.getProperty(JdbcDBClient.CONNECTION_USER); + String password = props.getProperty(JdbcDBClient.CONNECTION_PASSWD, ""); + String url = props.getProperty(JdbcDBClient.CONNECTION_URL); + if (driver == null || username == null || url == null) { + throw new SQLException("Missing connection information."); + } + + Connection conn = null; + + try { + Class.forName(driver); + + conn = DriverManager.getConnection(url, username, password); + Statement stmt = conn.createStatement(); + stmt.execute(sql); + System.out.println("Command \"" + sql + "\" successfully executed."); + } catch (ClassNotFoundException e) { + throw new SQLException("JDBC Driver class not found."); + } finally { + if (conn != null) { + System.out.println("Closing database connection."); + conn.close(); + } + } + } + + /** + * @param args + */ + public static void main(String[] args) { + + if (args.length == 0) { + usageMessage(); + System.exit(0); + } + + Properties props = new Properties(); + Properties fileprops = new Properties(); + String sql = null; + + // parse arguments + int argindex = 0; + while (args[argindex].startsWith("-")) { + if (args[argindex].compareTo("-P") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + String propfile = args[argindex]; + argindex++; + + Properties myfileprops = new Properties(); + try { + myfileprops.load(new FileInputStream(propfile)); + } catch (IOException e) { + System.out.println(e.getMessage()); + System.exit(0); + } + + // Issue #5 - remove call to stringPropertyNames to make compilable + // under Java 1.5 + for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { + String prop = (String) e.nextElement(); + + fileprops.setProperty(prop, myfileprops.getProperty(prop)); + } + + } else if (args[argindex].compareTo("-p") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + int eq = args[argindex].indexOf('='); + if (eq < 0) { + usageMessage(); + System.exit(0); + } + + String name = args[argindex].substring(0, eq); + String value = args[argindex].substring(eq + 1); + props.put(name, value); + argindex++; + } else if (args[argindex].compareTo("-c") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + sql = args[argindex++]; + } else { + System.out.println("Unknown option " + args[argindex]); + usageMessage(); + System.exit(0); + } + + if (argindex >= args.length) { + break; + } + } + + if (argindex != args.length) { + usageMessage(); + System.exit(0); + } + + // overwrite file properties with properties from the command line + + // Issue #5 - remove call to stringPropertyNames to make compilable under + // Java 1.5 + for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { + String prop = (String) e.nextElement(); + + fileprops.setProperty(prop, props.getProperty(prop)); + } + + if (sql == null) { + System.err.println("Missing command."); + usageMessage(); + System.exit(1); + } + + try { + executeCommand(fileprops, sql); + } catch (SQLException e) { + System.err.println("Error in executing command. " + e); + System.exit(1); + } + } + + /** + * Hidden constructor. + */ + private JdbcDBCli() { + super(); + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBClient.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBClient.java new file mode 100644 index 0000000..4916b9a --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBClient.java @@ -0,0 +1,534 @@ +/** + * Copyright (c) 2010 - 2016 Yahoo! Inc., 2016, 2019 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import site.ycsb.DB; +import site.ycsb.DBException; +import site.ycsb.ByteIterator; +import site.ycsb.Status; +import site.ycsb.StringByteIterator; + +import java.sql.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import site.ycsb.db.flavors.DBFlavor; + +/** + * A class that wraps a JDBC compliant database to allow it to be interfaced + * with YCSB. This class extends {@link DB} and implements the database + * interface used by YCSB client. + * + *
+ * Each client will have its own instance of this class. This client is not + * thread safe. + * + *
+ * This interface expects a schema ... All + * attributes are of type TEXT. All accesses are through the primary key. + * Therefore, only one index on the primary key is needed. + */ +public class JdbcDBClient extends DB { + + /** The class to use as the jdbc driver. */ + public static final String DRIVER_CLASS = "db.driver"; + + /** The URL to connect to the database. */ + public static final String CONNECTION_URL = "db.url"; + + /** The user name to use to connect to the database. */ + public static final String CONNECTION_USER = "db.user"; + + /** The password to use for establishing the connection. */ + public static final String CONNECTION_PASSWD = "db.passwd"; + + /** The batch size for batched inserts. Set to >0 to use batching */ + public static final String DB_BATCH_SIZE = "db.batchsize"; + + /** The JDBC fetch size hinted to the driver. */ + public static final String JDBC_FETCH_SIZE = "jdbc.fetchsize"; + + /** The JDBC connection auto-commit property for the driver. */ + public static final String JDBC_AUTO_COMMIT = "jdbc.autocommit"; + + public static final String JDBC_BATCH_UPDATES = "jdbc.batchupdateapi"; + + /** The name of the property for the number of fields in a record. */ + public static final String FIELD_COUNT_PROPERTY = "fieldcount"; + + /** Default number of fields in a record. */ + public static final String FIELD_COUNT_PROPERTY_DEFAULT = "10"; + + /** Representing a NULL value. */ + public static final String NULL_VALUE = "NULL"; + + /** The primary key in the user table. */ + public static final String PRIMARY_KEY = "YCSB_KEY"; + + /** The field name prefix in the table. */ + public static final String COLUMN_PREFIX = "FIELD"; + + /** SQL:2008 standard: FETCH FIRST n ROWS after the ORDER BY. */ + private boolean sqlansiScans = false; + /** SQL Server before 2012: TOP n after the SELECT. */ + private boolean sqlserverScans = false; + + private List conns; + private boolean initialized = false; + private Properties props; + private int jdbcFetchSize; + private int batchSize; + private boolean autoCommit; + private boolean batchUpdates; + private static final String DEFAULT_PROP = ""; + private ConcurrentMap cachedStatements; + private long numRowsInBatch = 0; + /** DB flavor defines DB-specific syntax and behavior for the + * particular database. Current database flavors are: {default, phoenix} */ + private DBFlavor dbFlavor; + + /** + * Ordered field information for insert and update statements. + */ + private static class OrderedFieldInfo { + private String fieldKeys; + private List fieldValues; + + OrderedFieldInfo(String fieldKeys, List fieldValues) { + this.fieldKeys = fieldKeys; + this.fieldValues = fieldValues; + } + + String getFieldKeys() { + return fieldKeys; + } + + List getFieldValues() { + return fieldValues; + } + } + + /** + * For the given key, returns what shard contains data for this key. + * + * @param key Data key to do operation on + * @return Shard index + */ + private int getShardIndexByKey(String key) { + int ret = Math.abs(key.hashCode()) % conns.size(); + return ret; + } + + /** + * For the given key, returns Connection object that holds connection to the + * shard that contains this key. + * + * @param key Data key to get information for + * @return Connection object + */ + private Connection getShardConnectionByKey(String key) { + return conns.get(getShardIndexByKey(key)); + } + + private void cleanupAllConnections() throws SQLException { + for (Connection conn : conns) { + if (!autoCommit) { + conn.commit(); + } + conn.close(); + } + } + + /** Returns parsed int value from the properties if set, otherwise returns -1. */ + private static int getIntProperty(Properties props, String key) throws DBException { + String valueStr = props.getProperty(key); + if (valueStr != null) { + try { + return Integer.parseInt(valueStr); + } catch (NumberFormatException nfe) { + System.err.println("Invalid " + key + " specified: " + valueStr); + throw new DBException(nfe); + } + } + return -1; + } + + /** Returns parsed boolean value from the properties if set, otherwise returns defaultVal. */ + private static boolean getBoolProperty(Properties props, String key, boolean defaultVal) { + String valueStr = props.getProperty(key); + if (valueStr != null) { + return Boolean.parseBoolean(valueStr); + } + return defaultVal; + } + + @Override + public void init() throws DBException { + if (initialized) { + System.err.println("Client connection already initialized."); + return; + } + props = getProperties(); + String urls = props.getProperty(CONNECTION_URL, DEFAULT_PROP); + String user = props.getProperty(CONNECTION_USER, DEFAULT_PROP); + String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP); + String driver = props.getProperty(DRIVER_CLASS); + + this.jdbcFetchSize = getIntProperty(props, JDBC_FETCH_SIZE); + this.batchSize = getIntProperty(props, DB_BATCH_SIZE); + + this.autoCommit = getBoolProperty(props, JDBC_AUTO_COMMIT, true); + this.batchUpdates = getBoolProperty(props, JDBC_BATCH_UPDATES, false); + + try { +// The SQL Syntax for Scan depends on the DB engine +// - SQL:2008 standard: FETCH FIRST n ROWS after the ORDER BY +// - SQL Server before 2012: TOP n after the SELECT +// - others (MySQL,MariaDB, PostgreSQL before 8.4) +// TODO: check product name and version rather than driver name + if (driver != null) { + if (driver.contains("sqlserver")) { + sqlserverScans = true; + sqlansiScans = false; + } + if (driver.contains("oracle")) { + sqlserverScans = false; + sqlansiScans = true; + } + if (driver.contains("postgres")) { + sqlserverScans = false; + sqlansiScans = true; + } + Class.forName(driver); + } + int shardCount = 0; + conns = new ArrayList(3); + // for a longer explanation see the README.md + // semicolons aren't present in JDBC urls, so we use them to delimit + // multiple JDBC connections to shard across. + final String[] urlArr = urls.split(";"); + for (String url : urlArr) { + System.out.println("Adding shard node URL: " + url); + Connection conn = DriverManager.getConnection(url, user, passwd); + + // Since there is no explicit commit method in the DB interface, all + // operations should auto commit, except when explicitly told not to + // (this is necessary in cases such as for PostgreSQL when running a + // scan workload with fetchSize) + conn.setAutoCommit(autoCommit); + + shardCount++; + conns.add(conn); + } + + System.out.println("Using shards: " + shardCount + ", batchSize:" + batchSize + ", fetchSize: " + jdbcFetchSize); + + cachedStatements = new ConcurrentHashMap(); + + this.dbFlavor = DBFlavor.fromJdbcUrl(urlArr[0]); + } catch (ClassNotFoundException e) { + System.err.println("Error in initializing the JDBS driver: " + e); + throw new DBException(e); + } catch (SQLException e) { + System.err.println("Error in database operation: " + e); + throw new DBException(e); + } catch (NumberFormatException e) { + System.err.println("Invalid value for fieldcount property. " + e); + throw new DBException(e); + } + + initialized = true; + } + + @Override + public void cleanup() throws DBException { + if (batchSize > 0) { + try { + // commit un-finished batches + for (PreparedStatement st : cachedStatements.values()) { + if (!st.getConnection().isClosed() && !st.isClosed() && (numRowsInBatch % batchSize != 0)) { + st.executeBatch(); + } + } + } catch (SQLException e) { + System.err.println("Error in cleanup execution. " + e); + throw new DBException(e); + } + } + + try { + cleanupAllConnections(); + } catch (SQLException e) { + System.err.println("Error in closing the connection. " + e); + throw new DBException(e); + } + } + + private PreparedStatement createAndCacheInsertStatement(StatementType insertType, String key) + throws SQLException { + String insert = dbFlavor.createInsertStatement(insertType, key); + PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(insert); + PreparedStatement stmt = cachedStatements.putIfAbsent(insertType, insertStatement); + if (stmt == null) { + return insertStatement; + } + return stmt; + } + + private PreparedStatement createAndCacheReadStatement(StatementType readType, String key) + throws SQLException { + String read = dbFlavor.createReadStatement(readType, key); + PreparedStatement readStatement = getShardConnectionByKey(key).prepareStatement(read); + PreparedStatement stmt = cachedStatements.putIfAbsent(readType, readStatement); + if (stmt == null) { + return readStatement; + } + return stmt; + } + + private PreparedStatement createAndCacheDeleteStatement(StatementType deleteType, String key) + throws SQLException { + String delete = dbFlavor.createDeleteStatement(deleteType, key); + PreparedStatement deleteStatement = getShardConnectionByKey(key).prepareStatement(delete); + PreparedStatement stmt = cachedStatements.putIfAbsent(deleteType, deleteStatement); + if (stmt == null) { + return deleteStatement; + } + return stmt; + } + + private PreparedStatement createAndCacheUpdateStatement(StatementType updateType, String key) + throws SQLException { + String update = dbFlavor.createUpdateStatement(updateType, key); + PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(update); + PreparedStatement stmt = cachedStatements.putIfAbsent(updateType, insertStatement); + if (stmt == null) { + return insertStatement; + } + return stmt; + } + + private PreparedStatement createAndCacheScanStatement(StatementType scanType, String key) + throws SQLException { + String select = dbFlavor.createScanStatement(scanType, key, sqlserverScans, sqlansiScans); + PreparedStatement scanStatement = getShardConnectionByKey(key).prepareStatement(select); + if (this.jdbcFetchSize > 0) { + scanStatement.setFetchSize(this.jdbcFetchSize); + } + PreparedStatement stmt = cachedStatements.putIfAbsent(scanType, scanStatement); + if (stmt == null) { + return scanStatement; + } + return stmt; + } + + @Override + public Status read(String tableName, String key, Set fields, Map result) { + try { + StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, "", getShardIndexByKey(key)); + PreparedStatement readStatement = cachedStatements.get(type); + if (readStatement == null) { + readStatement = createAndCacheReadStatement(type, key); + } + readStatement.setString(1, key); + ResultSet resultSet = readStatement.executeQuery(); + if (!resultSet.next()) { + resultSet.close(); + return Status.NOT_FOUND; + } + if (result != null && fields != null) { + for (String field : fields) { + String value = resultSet.getString(field); + result.put(field, new StringByteIterator(value)); + } + } + resultSet.close(); + return Status.OK; + } catch (SQLException e) { + System.err.println("Error in processing read of table " + tableName + ": " + e); + return Status.ERROR; + } + } + + @Override + public Status scan(String tableName, String startKey, int recordcount, Set fields, + Vector> result) { + try { + StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, "", getShardIndexByKey(startKey)); + PreparedStatement scanStatement = cachedStatements.get(type); + if (scanStatement == null) { + scanStatement = createAndCacheScanStatement(type, startKey); + } + // SQL Server TOP syntax is at first + if (sqlserverScans) { + scanStatement.setInt(1, recordcount); + scanStatement.setString(2, startKey); + // FETCH FIRST and LIMIT are at the end + } else { + scanStatement.setString(1, startKey); + scanStatement.setInt(2, recordcount); + } + ResultSet resultSet = scanStatement.executeQuery(); + for (int i = 0; i < recordcount && resultSet.next(); i++) { + if (result != null && fields != null) { + HashMap values = new HashMap(); + for (String field : fields) { + String value = resultSet.getString(field); + values.put(field, new StringByteIterator(value)); + } + result.add(values); + } + } + resultSet.close(); + return Status.OK; + } catch (SQLException e) { + System.err.println("Error in processing scan of table: " + tableName + e); + return Status.ERROR; + } + } + + @Override + public Status update(String tableName, String key, Map values) { + try { + int numFields = values.size(); + OrderedFieldInfo fieldInfo = getFieldInfo(values); + StatementType type = new StatementType(StatementType.Type.UPDATE, tableName, + numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key)); + PreparedStatement updateStatement = cachedStatements.get(type); + if (updateStatement == null) { + updateStatement = createAndCacheUpdateStatement(type, key); + } + int index = 1; + for (String value: fieldInfo.getFieldValues()) { + updateStatement.setString(index++, value); + } + updateStatement.setString(index, key); + int result = updateStatement.executeUpdate(); + if (result == 1) { + return Status.OK; + } + return Status.UNEXPECTED_STATE; + } catch (SQLException e) { + System.err.println("Error in processing update to table: " + tableName + e); + return Status.ERROR; + } + } + + @Override + public Status insert(String tableName, String key, Map values) { + try { + int numFields = values.size(); + OrderedFieldInfo fieldInfo = getFieldInfo(values); + StatementType type = new StatementType(StatementType.Type.INSERT, tableName, + numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key)); + PreparedStatement insertStatement = cachedStatements.get(type); + if (insertStatement == null) { + insertStatement = createAndCacheInsertStatement(type, key); + } + insertStatement.setString(1, key); + int index = 2; + for (String value: fieldInfo.getFieldValues()) { + insertStatement.setString(index++, value); + } + // Using the batch insert API + if (batchUpdates) { + insertStatement.addBatch(); + // Check for a sane batch size + if (batchSize > 0) { + // Commit the batch after it grows beyond the configured size + if (++numRowsInBatch % batchSize == 0) { + int[] results = insertStatement.executeBatch(); + for (int r : results) { + // Acceptable values are 1 and SUCCESS_NO_INFO (-2) from reWriteBatchedInserts=true + if (r != 1 && r != -2) { + return Status.ERROR; + } + } + // If autoCommit is off, make sure we commit the batch + if (!autoCommit) { + getShardConnectionByKey(key).commit(); + } + return Status.OK; + } // else, the default value of -1 or a nonsense. Treat it as an infinitely large batch. + } // else, we let the batch accumulate + // Added element to the batch, potentially committing the batch too. + return Status.BATCHED_OK; + } else { + // Normal update + int result = insertStatement.executeUpdate(); + // If we are not autoCommit, we might have to commit now + if (!autoCommit) { + // Let updates be batcher locally + if (batchSize > 0) { + if (++numRowsInBatch % batchSize == 0) { + // Send the batch of updates + getShardConnectionByKey(key).commit(); + } + // uhh + return Status.OK; + } else { + // Commit each update + getShardConnectionByKey(key).commit(); + } + } + if (result == 1) { + return Status.OK; + } + } + return Status.UNEXPECTED_STATE; + } catch (SQLException e) { + System.err.println("Error in processing insert to table: " + tableName + e); + return Status.ERROR; + } + } + + @Override + public Status delete(String tableName, String key) { + try { + StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, "", getShardIndexByKey(key)); + PreparedStatement deleteStatement = cachedStatements.get(type); + if (deleteStatement == null) { + deleteStatement = createAndCacheDeleteStatement(type, key); + } + deleteStatement.setString(1, key); + int result = deleteStatement.executeUpdate(); + if (result == 1) { + return Status.OK; + } + return Status.UNEXPECTED_STATE; + } catch (SQLException e) { + System.err.println("Error in processing delete to table: " + tableName + e); + return Status.ERROR; + } + } + + private OrderedFieldInfo getFieldInfo(Map values) { + String fieldKeys = ""; + List fieldValues = new ArrayList<>(); + int count = 0; + for (Map.Entry entry : values.entrySet()) { + fieldKeys += entry.getKey(); + if (count < values.size() - 1) { + fieldKeys += ","; + } + fieldValues.add(count, entry.getValue().toString()); + count++; + } + + return new OrderedFieldInfo(fieldKeys, fieldValues); + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCreateTable.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCreateTable.java new file mode 100644 index 0000000..8e79b88 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/JdbcDBCreateTable.java @@ -0,0 +1,224 @@ +/** + * Copyright (c) 2010 - 2016 Yahoo! Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +import java.io.FileInputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Enumeration; +import java.util.Properties; + +/** + * Utility class to create the table to be used by the benchmark. + * + * @author sudipto + */ +public final class JdbcDBCreateTable { + + private static void usageMessage() { + System.out.println("Create Table Client. Options:"); + System.out.println(" -p key=value properties defined."); + System.out.println(" -P location of the properties file to load."); + System.out.println(" -n name of the table."); + System.out.println(" -f number of fields (default 10)."); + } + + private static void createTable(Properties props, String tablename) throws SQLException { + String driver = props.getProperty(JdbcDBClient.DRIVER_CLASS); + String username = props.getProperty(JdbcDBClient.CONNECTION_USER); + String password = props.getProperty(JdbcDBClient.CONNECTION_PASSWD, ""); + String url = props.getProperty(JdbcDBClient.CONNECTION_URL); + int fieldcount = Integer.parseInt(props.getProperty(JdbcDBClient.FIELD_COUNT_PROPERTY, + JdbcDBClient.FIELD_COUNT_PROPERTY_DEFAULT)); + + if (driver == null || username == null || url == null) { + throw new SQLException("Missing connection information."); + } + + Connection conn = null; + + try { + Class.forName(driver); + + conn = DriverManager.getConnection(url, username, password); + Statement stmt = conn.createStatement(); + + StringBuilder sql = new StringBuilder("DROP TABLE IF EXISTS "); + sql.append(tablename); + sql.append(";"); + + stmt.execute(sql.toString()); + + sql = new StringBuilder("CREATE TABLE "); + sql.append(tablename); + sql.append(" (YCSB_KEY VARCHAR PRIMARY KEY"); + + for (int idx = 0; idx < fieldcount; idx++) { + sql.append(", FIELD"); + sql.append(idx); + sql.append(" TEXT"); + } + sql.append(");"); + + stmt.execute(sql.toString()); + + System.out.println("Table " + tablename + " created.."); + } catch (ClassNotFoundException e) { + throw new SQLException("JDBC Driver class not found."); + } finally { + if (conn != null) { + System.out.println("Closing database connection."); + conn.close(); + } + } + } + + /** + * @param args + */ + public static void main(String[] args) { + + if (args.length == 0) { + usageMessage(); + System.exit(0); + } + + String tablename = null; + int fieldcount = -1; + Properties props = new Properties(); + Properties fileprops = new Properties(); + + // parse arguments + int argindex = 0; + while (args[argindex].startsWith("-")) { + if (args[argindex].compareTo("-P") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + String propfile = args[argindex]; + argindex++; + + Properties myfileprops = new Properties(); + try { + myfileprops.load(new FileInputStream(propfile)); + } catch (IOException e) { + System.out.println(e.getMessage()); + System.exit(0); + } + + // Issue #5 - remove call to stringPropertyNames to make compilable + // under Java 1.5 + for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { + String prop = (String) e.nextElement(); + + fileprops.setProperty(prop, myfileprops.getProperty(prop)); + } + + } else if (args[argindex].compareTo("-p") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + int eq = args[argindex].indexOf('='); + if (eq < 0) { + usageMessage(); + System.exit(0); + } + + String name = args[argindex].substring(0, eq); + String value = args[argindex].substring(eq + 1); + props.put(name, value); + argindex++; + } else if (args[argindex].compareTo("-n") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + tablename = args[argindex++]; + } else if (args[argindex].compareTo("-f") == 0) { + argindex++; + if (argindex >= args.length) { + usageMessage(); + System.exit(0); + } + try { + fieldcount = Integer.parseInt(args[argindex++]); + } catch (NumberFormatException e) { + System.err.println("Invalid number for field count"); + usageMessage(); + System.exit(1); + } + } else { + System.out.println("Unknown option " + args[argindex]); + usageMessage(); + System.exit(0); + } + + if (argindex >= args.length) { + break; + } + } + + if (argindex != args.length) { + usageMessage(); + System.exit(0); + } + + // overwrite file properties with properties from the command line + + // Issue #5 - remove call to stringPropertyNames to make compilable under + // Java 1.5 + for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { + String prop = (String) e.nextElement(); + + fileprops.setProperty(prop, props.getProperty(prop)); + } + + props = fileprops; + + if (tablename == null) { + System.err.println("table name missing."); + usageMessage(); + System.exit(1); + } + + if (fieldcount > 0) { + props.setProperty(JdbcDBClient.FIELD_COUNT_PROPERTY, String.valueOf(fieldcount)); + } + + try { + createTable(props, tablename); + } catch (SQLException e) { + System.err.println("Error in creating table. " + e); + System.exit(1); + } + } + + /** + * Hidden constructor. + */ + private JdbcDBCreateTable() { + super(); + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/StatementType.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/StatementType.java new file mode 100644 index 0000000..95700ed --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/StatementType.java @@ -0,0 +1,110 @@ +/** + * Copyright (c) 2010 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db; + +/** + * The statement type for the prepared statements. + */ +public class StatementType { + + enum Type { + INSERT(1), DELETE(2), READ(3), UPDATE(4), SCAN(5); + + private final int internalType; + + private Type(int type) { + internalType = type; + } + + int getHashCode() { + final int prime = 31; + int result = 1; + result = prime * result + internalType; + return result; + } + } + + private Type type; + private int shardIndex; + private int numFields; + private String tableName; + private String fieldString; + + public StatementType(Type type, String tableName, int numFields, String fieldString, int shardIndex) { + this.type = type; + this.tableName = tableName; + this.numFields = numFields; + this.fieldString = fieldString; + this.shardIndex = shardIndex; + } + + public String getTableName() { + return tableName; + } + + public String getFieldString() { + return fieldString; + } + + public int getNumFields() { + return numFields; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + numFields + 100 * shardIndex; + result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); + result = prime * result + ((type == null) ? 0 : type.getHashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StatementType other = (StatementType) obj; + if (numFields != other.numFields) { + return false; + } + if (shardIndex != other.shardIndex) { + return false; + } + if (tableName == null) { + if (other.tableName != null) { + return false; + } + } else if (!tableName.equals(other.tableName)) { + return false; + } + if (type != other.type) { + return false; + } + if (!fieldString.equals(other.fieldString)) { + return false; + } + return true; + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DBFlavor.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DBFlavor.java new file mode 100644 index 0000000..a8b9e0b --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DBFlavor.java @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2016, 2019 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db.flavors; + +import site.ycsb.db.StatementType; + +/** + * DBFlavor captures minor differences in syntax and behavior among JDBC implementations and SQL + * dialects. This class also acts as a factory to instantiate concrete flavors based on the JDBC URL. + */ +public abstract class DBFlavor { + + enum DBName { + DEFAULT, + PHOENIX + } + + private final DBName dbName; + + public DBFlavor(DBName dbName) { + this.dbName = dbName; + } + + public static DBFlavor fromJdbcUrl(String url) { + if (url.startsWith("jdbc:phoenix")) { + return new PhoenixDBFlavor(); + } + return new DefaultDBFlavor(); + } + + /** + * Create and return a SQL statement for inserting data. + */ + public abstract String createInsertStatement(StatementType insertType, String key); + + /** + * Create and return a SQL statement for reading data. + */ + public abstract String createReadStatement(StatementType readType, String key); + + /** + * Create and return a SQL statement for deleting data. + */ + public abstract String createDeleteStatement(StatementType deleteType, String key); + + /** + * Create and return a SQL statement for updating data. + */ + public abstract String createUpdateStatement(StatementType updateType, String key); + + /** + * Create and return a SQL statement for scanning data. + */ + public abstract String createScanStatement(StatementType scanType, String key, + boolean sqlserverScans, boolean sqlansiScans); +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DefaultDBFlavor.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DefaultDBFlavor.java new file mode 100644 index 0000000..d32417f --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/DefaultDBFlavor.java @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2016, 2019 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db.flavors; + +import site.ycsb.db.JdbcDBClient; +import site.ycsb.db.StatementType; + +/** + * A default flavor for relational databases. + */ +public class DefaultDBFlavor extends DBFlavor { + public DefaultDBFlavor() { + super(DBName.DEFAULT); + } + public DefaultDBFlavor(DBName dbName) { + super(dbName); + } + + @Override + public String createInsertStatement(StatementType insertType, String key) { + StringBuilder insert = new StringBuilder("INSERT INTO "); + insert.append(insertType.getTableName()); + insert.append(" (" + JdbcDBClient.PRIMARY_KEY + "," + insertType.getFieldString() + ")"); + insert.append(" VALUES(?"); + for (int i = 0; i < insertType.getNumFields(); i++) { + insert.append(",?"); + } + insert.append(")"); + return insert.toString(); + } + + @Override + public String createReadStatement(StatementType readType, String key) { + StringBuilder read = new StringBuilder("SELECT * FROM "); + read.append(readType.getTableName()); + read.append(" WHERE "); + read.append(JdbcDBClient.PRIMARY_KEY); + read.append(" = "); + read.append("?"); + return read.toString(); + } + + @Override + public String createDeleteStatement(StatementType deleteType, String key) { + StringBuilder delete = new StringBuilder("DELETE FROM "); + delete.append(deleteType.getTableName()); + delete.append(" WHERE "); + delete.append(JdbcDBClient.PRIMARY_KEY); + delete.append(" = ?"); + return delete.toString(); + } + + @Override + public String createUpdateStatement(StatementType updateType, String key) { + String[] fieldKeys = updateType.getFieldString().split(","); + StringBuilder update = new StringBuilder("UPDATE "); + update.append(updateType.getTableName()); + update.append(" SET "); + for (int i = 0; i < fieldKeys.length; i++) { + update.append(fieldKeys[i]); + update.append("=?"); + if (i < fieldKeys.length - 1) { + update.append(", "); + } + } + update.append(" WHERE "); + update.append(JdbcDBClient.PRIMARY_KEY); + update.append(" = ?"); + return update.toString(); + } + + @Override + public String createScanStatement(StatementType scanType, String key, boolean sqlserverScans, boolean sqlansiScans) { + StringBuilder select; + if (sqlserverScans) { + select = new StringBuilder("SELECT TOP (?) * FROM "); + } else { + select = new StringBuilder("SELECT * FROM "); + } + select.append(scanType.getTableName()); + select.append(" WHERE "); + select.append(JdbcDBClient.PRIMARY_KEY); + select.append(" >= ?"); + select.append(" ORDER BY "); + select.append(JdbcDBClient.PRIMARY_KEY); + if (!sqlserverScans) { + if (sqlansiScans) { + select.append(" FETCH FIRST ? ROWS ONLY"); + } else { + select.append(" LIMIT ?"); + } + } + return select.toString(); + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/PhoenixDBFlavor.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/PhoenixDBFlavor.java new file mode 100644 index 0000000..3f1c811 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/PhoenixDBFlavor.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db.flavors; + +import site.ycsb.db.JdbcDBClient; +import site.ycsb.db.StatementType; + +/** + * Database flavor for Apache Phoenix. Captures syntax differences used by Phoenix. + */ +public class PhoenixDBFlavor extends DefaultDBFlavor { + public PhoenixDBFlavor() { + super(DBName.PHOENIX); + } + + @Override + public String createInsertStatement(StatementType insertType, String key) { + // Phoenix uses UPSERT syntax + StringBuilder insert = new StringBuilder("UPSERT INTO "); + insert.append(insertType.getTableName()); + insert.append(" (" + JdbcDBClient.PRIMARY_KEY + "," + insertType.getFieldString() + ")"); + insert.append(" VALUES(?"); + for (int i = 0; i < insertType.getNumFields(); i++) { + insert.append(",?"); + } + insert.append(")"); + return insert.toString(); + } + + @Override + public String createUpdateStatement(StatementType updateType, String key) { + // Phoenix doesn't have UPDATE semantics, just re-use UPSERT VALUES on the specific columns + String[] fieldKeys = updateType.getFieldString().split(","); + StringBuilder update = new StringBuilder("UPSERT INTO "); + update.append(updateType.getTableName()); + update.append(" ("); + // Each column to update + for (int i = 0; i < fieldKeys.length; i++) { + update.append(fieldKeys[i]).append(","); + } + // And then set the primary key column + update.append(JdbcDBClient.PRIMARY_KEY).append(") VALUES("); + // Add an unbound param for each column to update + for (int i = 0; i < fieldKeys.length; i++) { + update.append("?, "); + } + // Then the primary key column's value + update.append("?)"); + return update.toString(); + } +} diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/package-info.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/package-info.java new file mode 100644 index 0000000..2b24640 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/flavors/package-info.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +/** + * This package contains a collection of database-specific overrides. This accounts for the variance + * that can be present where JDBC does not explicitly define what a database must do or when a + * database has a non-standard SQL implementation. + */ +package site.ycsb.db.flavors; diff --git a/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/package-info.java b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/package-info.java new file mode 100644 index 0000000..bdefd44 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/java/site/ycsb/db/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014 - 2016, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for stores that can be accessed via JDBC. + */ +package site.ycsb.db; + diff --git a/ycsb-mongodb/jdbc/src/main/resources/sql/README.md b/ycsb-mongodb/jdbc/src/main/resources/sql/README.md new file mode 100644 index 0000000..ec41b86 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/resources/sql/README.md @@ -0,0 +1,18 @@ + +Contains all the SQL statements used by the JDBC client. diff --git a/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.mysql b/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.mysql new file mode 100644 index 0000000..a88a73a --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.mysql @@ -0,0 +1,27 @@ +-- Copyright (c) 2015 YCSB contributors. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); you +-- may not use this file except in compliance with the License. You +-- may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +-- implied. See the License for the specific language governing +-- permissions and limitations under the License. See accompanying +-- LICENSE file. + +-- Creates a Table. + +-- Drop the table if it exists; +DROP TABLE IF EXISTS usertable; + +-- Create the user table with 5 fields. +CREATE TABLE usertable(YCSB_KEY VARCHAR (255) PRIMARY KEY, + FIELD0 TEXT, FIELD1 TEXT, + FIELD2 TEXT, FIELD3 TEXT, + FIELD4 TEXT, FIELD5 TEXT, + FIELD6 TEXT, FIELD7 TEXT, + FIELD8 TEXT, FIELD9 TEXT); diff --git a/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.sql b/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.sql new file mode 100644 index 0000000..33158ac --- /dev/null +++ b/ycsb-mongodb/jdbc/src/main/resources/sql/create_table.sql @@ -0,0 +1,27 @@ +-- Copyright (c) 2015 YCSB contributors. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); you +-- may not use this file except in compliance with the License. You +-- may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +-- implied. See the License for the specific language governing +-- permissions and limitations under the License. See accompanying +-- LICENSE file. + +-- Creates a Table. + +-- Drop the table if it exists; +DROP TABLE IF EXISTS usertable; + +-- Create the user table with 5 fields. +CREATE TABLE usertable(YCSB_KEY VARCHAR PRIMARY KEY, + FIELD0 VARCHAR, FIELD1 VARCHAR, + FIELD2 VARCHAR, FIELD3 VARCHAR, + FIELD4 VARCHAR, FIELD5 VARCHAR, + FIELD6 VARCHAR, FIELD7 VARCHAR, + FIELD8 VARCHAR, FIELD9 VARCHAR); diff --git a/ycsb-mongodb/jdbc/src/test/java/site/ycsb/db/JdbcDBClientTest.java b/ycsb-mongodb/jdbc/src/test/java/site/ycsb/db/JdbcDBClientTest.java new file mode 100644 index 0000000..a7d1e98 --- /dev/null +++ b/ycsb-mongodb/jdbc/src/test/java/site/ycsb/db/JdbcDBClientTest.java @@ -0,0 +1,393 @@ +/** + * Copyright (c) 2015 - 2016 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package site.ycsb.db; + +import static org.junit.Assert.*; + +import site.ycsb.ByteIterator; +import site.ycsb.DBException; +import site.ycsb.StringByteIterator; +import org.junit.*; + +import java.sql.*; +import java.util.HashMap; +import java.util.Map; +import java.util.HashSet; +import java.util.Set; +import java.util.Properties; +import java.util.Vector; + +public class JdbcDBClientTest { + private static final String TEST_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver"; + private static final String TEST_DB_URL = "jdbc:hsqldb:mem:ycsb"; + private static final String TEST_DB_USER = "sa"; + private static final String TABLE_NAME = "USERTABLE"; + private static final int FIELD_LENGTH = 32; + private static final String FIELD_PREFIX = "FIELD"; + private static final String KEY_PREFIX = "user"; + private static final String KEY_FIELD = "YCSB_KEY"; + private static final int NUM_FIELDS = 3; + + private static Connection jdbcConnection = null; + private static JdbcDBClient jdbcDBClient = null; + + @BeforeClass + public static void setup() { + setupWithBatch(1, true); + } + + public static void setupWithBatch(int batchSize, boolean autoCommit) { + try { + jdbcConnection = DriverManager.getConnection(TEST_DB_URL); + jdbcDBClient = new JdbcDBClient(); + + Properties p = new Properties(); + p.setProperty(JdbcDBClient.CONNECTION_URL, TEST_DB_URL); + p.setProperty(JdbcDBClient.DRIVER_CLASS, TEST_DB_DRIVER); + p.setProperty(JdbcDBClient.CONNECTION_USER, TEST_DB_USER); + p.setProperty(JdbcDBClient.DB_BATCH_SIZE, Integer.toString(batchSize)); + p.setProperty(JdbcDBClient.JDBC_BATCH_UPDATES, "true"); + p.setProperty(JdbcDBClient.JDBC_AUTO_COMMIT, Boolean.toString(autoCommit)); + + jdbcDBClient.setProperties(p); + jdbcDBClient.init(); + } catch (SQLException e) { + e.printStackTrace(); + fail("Could not create local Database"); + } catch (DBException e) { + e.printStackTrace(); + fail("Could not create JdbcDBClient instance"); + } + } + + @AfterClass + public static void teardown() { + try { + if (jdbcConnection != null) { + jdbcConnection.close(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + + try { + if (jdbcDBClient != null) { + jdbcDBClient.cleanup(); + } + } catch (DBException e) { + e.printStackTrace(); + } + } + + @Before + public void prepareTest() { + try { + DatabaseMetaData metaData = jdbcConnection.getMetaData(); + ResultSet tableResults = metaData.getTables(null, null, TABLE_NAME, null); + if (tableResults.next()) { + // If the table already exists, just truncate it + jdbcConnection.prepareStatement( + String.format("TRUNCATE TABLE %s", TABLE_NAME) + ).execute(); + } else { + // If the table does not exist then create it + StringBuilder createString = new StringBuilder( + String.format("CREATE TABLE %s (%s VARCHAR(100) PRIMARY KEY", TABLE_NAME, KEY_FIELD) + ); + for (int i = 0; i < NUM_FIELDS; i++) { + createString.append( + String.format(", %s%d VARCHAR(100)", FIELD_PREFIX, i) + ); + } + createString.append(")"); + jdbcConnection.prepareStatement(createString.toString()).execute(); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("Failed to prepare test"); + } + } + + /* + This is a copy of buildDeterministicValue() from core:site.ycsb.workloads.CoreWorkload.java. + That method is neither public nor static so we need a copy. + */ + private String buildDeterministicValue(String key, String fieldkey) { + int size = FIELD_LENGTH; + StringBuilder sb = new StringBuilder(size); + sb.append(key); + sb.append(':'); + sb.append(fieldkey); + while (sb.length() < size) { + sb.append(':'); + sb.append(sb.toString().hashCode()); + } + sb.setLength(size); + + return sb.toString(); + } + + /* + Inserts a row of deterministic values for the given insertKey using the jdbcDBClient. + */ + private HashMap insertRow(String insertKey) { + HashMap insertMap = new HashMap(); + for (int i = 0; i < 3; i++) { + insertMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue(insertKey, FIELD_PREFIX + i))); + } + jdbcDBClient.insert(TABLE_NAME, insertKey, insertMap); + + return insertMap; + } + + @Test + public void insertTest() { + try { + String insertKey = "user0"; + HashMap insertMap = insertRow(insertKey); + + ResultSet resultSet = jdbcConnection.prepareStatement( + String.format("SELECT * FROM %s", TABLE_NAME) + ).executeQuery(); + + // Check we have a result Row + assertTrue(resultSet.next()); + // Check that all the columns have expected values + assertEquals(resultSet.getString(KEY_FIELD), insertKey); + for (int i = 0; i < 3; i++) { + assertEquals(resultSet.getString(FIELD_PREFIX + i), insertMap.get(FIELD_PREFIX + i).toString()); + } + // Check that we do not have any more rows + assertFalse(resultSet.next()); + + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + fail("Failed insertTest"); + } + } + + @Test + public void updateTest() { + try { + String preupdateString = "preupdate"; + StringBuilder fauxInsertString = new StringBuilder( + String.format("INSERT INTO %s VALUES(?", TABLE_NAME) + ); + for (int i = 0; i < NUM_FIELDS; i++) { + fauxInsertString.append(",?"); + } + fauxInsertString.append(")"); + + PreparedStatement fauxInsertStatement = jdbcConnection.prepareStatement(fauxInsertString.toString()); + for (int i = 2; i < NUM_FIELDS + 2; i++) { + fauxInsertStatement.setString(i, preupdateString); + } + + fauxInsertStatement.setString(1, "user0"); + fauxInsertStatement.execute(); + fauxInsertStatement.setString(1, "user1"); + fauxInsertStatement.execute(); + fauxInsertStatement.setString(1, "user2"); + fauxInsertStatement.execute(); + + HashMap updateMap = new HashMap(); + for (int i = 0; i < 3; i++) { + updateMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue("user1", FIELD_PREFIX + i))); + } + + jdbcDBClient.update(TABLE_NAME, "user1", updateMap); + + ResultSet resultSet = jdbcConnection.prepareStatement( + String.format("SELECT * FROM %s ORDER BY %s", TABLE_NAME, KEY_FIELD) + ).executeQuery(); + + // Ensure that user0 record was not changed + resultSet.next(); + assertEquals("Assert first row key is user0", resultSet.getString(KEY_FIELD), "user0"); + for (int i = 0; i < 3; i++) { + assertEquals("Assert first row fields contain preupdateString", resultSet.getString(FIELD_PREFIX + i), preupdateString); + } + + // Check that all the columns have expected values for user1 record + resultSet.next(); + assertEquals(resultSet.getString(KEY_FIELD), "user1"); + for (int i = 0; i < 3; i++) { + assertEquals(resultSet.getString(FIELD_PREFIX + i), updateMap.get(FIELD_PREFIX + i).toString()); + } + + // Ensure that user2 record was not changed + resultSet.next(); + assertEquals("Assert third row key is user2", resultSet.getString(KEY_FIELD), "user2"); + for (int i = 0; i < 3; i++) { + assertEquals("Assert third row fields contain preupdateString", resultSet.getString(FIELD_PREFIX + i), preupdateString); + } + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + fail("Failed updateTest"); + } + } + + @Test + public void readTest() { + String insertKey = "user0"; + HashMap insertMap = insertRow(insertKey); + Set readFields = new HashSet(); + HashMap readResultMap = new HashMap(); + + // Test reading a single field + readFields.add("FIELD0"); + jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap); + assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); + for (String field: readFields) { + assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); + } + + readResultMap = new HashMap(); + + // Test reading all fields + readFields.add("FIELD1"); + readFields.add("FIELD2"); + jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap); + assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); + for (String field: readFields) { + assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); + } + } + + @Test + public void deleteTest() { + try { + insertRow("user0"); + String deleteKey = "user1"; + insertRow(deleteKey); + insertRow("user2"); + + jdbcDBClient.delete(TABLE_NAME, deleteKey); + + ResultSet resultSet = jdbcConnection.prepareStatement( + String.format("SELECT * FROM %s", TABLE_NAME) + ).executeQuery(); + + int totalRows = 0; + while (resultSet.next()) { + assertNotEquals("Assert this is not the deleted row key", deleteKey, resultSet.getString(KEY_FIELD)); + totalRows++; + } + // Check we do not have a result Row + assertEquals("Assert we ended with the correct number of rows", totalRows, 2); + + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + fail("Failed deleteTest"); + } + } + + @Test + public void scanTest() throws SQLException { + Map> keyMap = new HashMap>(); + for (int i = 0; i < 5; i++) { + String insertKey = KEY_PREFIX + i; + keyMap.put(insertKey, insertRow(insertKey)); + } + Set fieldSet = new HashSet(); + fieldSet.add("FIELD0"); + fieldSet.add("FIELD1"); + int startIndex = 1; + int resultRows = 3; + + Vector> resultVector = new Vector>(); + jdbcDBClient.scan(TABLE_NAME, KEY_PREFIX + startIndex, resultRows, fieldSet, resultVector); + + // Check the resultVector is the correct size + assertEquals("Assert the correct number of results rows were returned", resultRows, resultVector.size()); + // Check each vector row to make sure we have the correct fields + int testIndex = startIndex; + for (Map result: resultVector) { + assertEquals("Assert that this row has the correct number of fields", fieldSet.size(), result.size()); + for (String field: fieldSet) { + assertEquals("Assert this field is correct in this row", keyMap.get(KEY_PREFIX + testIndex).get(field).toString(), result.get(field).toString()); + } + testIndex++; + } + } + + @Test + public void insertBatchTest() throws DBException { + insertBatchTest(20); + } + + @Test + public void insertPartialBatchTest() throws DBException { + insertBatchTest(19); + } + + public void insertBatchTest(int numRows) throws DBException { + teardown(); + setupWithBatch(10, false); + try { + String insertKey = "user0"; + HashMap insertMap = insertRow(insertKey); + assertEquals(3, insertMap.size()); + + ResultSet resultSet = jdbcConnection.prepareStatement( + String.format("SELECT * FROM %s", TABLE_NAME) + ).executeQuery(); + + // Check we do not have a result Row (because batch is not full yet) + assertFalse(resultSet.next()); + // insert more rows, completing 1 batch (still results are partial). + for (int i = 1; i < numRows; i++) { + insertMap = insertRow("user" + i); + } + + // + assertNumRows(10 * (numRows / 10)); + + // call cleanup, which should insert the partial batch + jdbcDBClient.cleanup(); + // Prevent a teardown() from printing an error + jdbcDBClient = null; + + // Check that we have all rows + assertNumRows(numRows); + + } catch (SQLException e) { + e.printStackTrace(); + fail("Failed insertBatchTest"); + } finally { + teardown(); // for next tests + setup(); + } + } + + private void assertNumRows(long numRows) throws SQLException { + ResultSet resultSet = jdbcConnection.prepareStatement( + String.format("SELECT * FROM %s", TABLE_NAME) + ).executeQuery(); + + for (int i = 0; i < numRows; i++) { + assertTrue("expecting " + numRows + " results, received only " + i, resultSet.next()); + } + assertFalse("expecting " + numRows + " results, received more", resultSet.next()); + + resultSet.close(); + } +} diff --git a/ycsb-mongodb/pom.xml b/ycsb-mongodb/pom.xml index f7d90dc..e49cfcb 100644 --- a/ycsb-mongodb/pom.xml +++ b/ycsb-mongodb/pom.xml @@ -58,6 +58,7 @@ 4.7.0 1.1.8.4 1.5.0-4 + 2.1.1 UTF-8 true true @@ -65,6 +66,7 @@ core + jdbc mongodb