[WIP] config cleanup v2

.gitignore

@@ -19,4 +19,12 @@ results
 *~
 
 # maven output
-target
+target
+
+# IDE
+.idea
+*.iml
+.vscode
+
+# Mac
+.DS_Store

.travis.yml

@@ -1,168 +1,69 @@
-dist: trusty
+git:
+  depth: 3
 sudo: required
-group: deprecated-2017Q2
-
-# lib contains jar files to support running tests
-cache:
-  directories:
-    - lib
-
-language: java
+services:
+  - docker
+language: java # default is oracle JDK 1.8.0_151
 jdk:
   - oraclejdk8
-  - openjdk8
-
-# oraclejdk9 failures - https://github.com/oltpbenchmark/oltpbench/issues/137
-#matrix:
-#  allow_failures:
-#    - jdk: oraclejdk9
-
-# Currently oraclejdk8 is mapped to DB=mysql and the
-# others are DB=postgres.
-#
-# a test env of: - DB=mysql  TEST=tatp
-# duplicated for postgres would of been nice however
-# attempting to limit non-junit tests to single jdk with
-# matrix:
-#   exclude:
-#     - jdk: oraclejdk8
-#       env: DB=mysql
-#     - jdk: oraclejdk9
-#       env: DB=mysql
-#     - jdk: oraclejdk8
-#       env: DB=postgres
-#     - jdk: oraclejdk9
-#       env: DB=postgres
-# Failed to exclude any elements.
-
-# Possibilities for future:
-# matrix:
-#   include:
-#    - addons:
-#      mariadb: 5.5
-#    - addons:
-#      mariadb: 10.0
-#    - addons:
-#      mariadb: 10.1
-#    - addons:
-#      mariadb: 10.2
-#    - addons:
-#      postgresql: 9.4
-#    - addons:
-#      postgresql: 9.5
-#    - addons:
-#      postgresql: 9.6
-##    - addons:
-##      postgresql: 9.7
-# https://docs.travis-ci.com/user/database-setup/#Using-a-different-PostgreSQL-Version
-# MariaDB wasn't supported on trusty-containers at time of writing
-# This syntax may not be correct.
-
 addons:
-  postgresql: 9.6
-
+  apt:
+    packages:
+      - "python3"
+      - "python3-pip"
+# TODO: build matrix for different database and benchmarks
+# https://docs.travis-ci.com/user/environment-variables/#Defining-Multiple-Variables-per-Item
 env:
-  - TEST=junit
-  - TEST=epinions
-  - TEST=tatp
-  - TEST=tpcc
-  - TEST=voter
-  - TEST=auctionmark
-  - TEST=wikipedia
-  - TEST=ycsb
-  - TEST=seats
-  - TEST=sibench
-  - TEST=noop
-  - TEST=smallbank
-  - TEST=twitter
-  - TEST=resourcestresser
-
-# Missing loader com.oltpbenchmark.benchmarks.resourcestresser.ResourceStresserBenchmark.makeLoaderImpl(ResourceStresserBenchmark.java:58)
-# -> org.apache.commons.lang.NotImplementedException: Code is not implemented
-#  - TEST=resourcestresser
-
-# Missing sample config file
-#  - TEST=chbenchmark
-#
-# Missing sample config
-#  - TEST=hyadapt
-#
-# Not fully implemented
-#  - TEST=jpab
-#
-# Works only for MySQL at this time
-#  - TEST=linkbench
-#
-
-# While mariadb addon isn't used. Remove if using a addon: mariadb
-services:
-  - mysql
-
-# Increase the size of ramdisk on sudo enabled infrastructure
-# https://github.com/travis-ci/travis-ci/issues/9036#issuecomment-357189213
+# --- begin tpcc with different databases ---
+  - BENCH=tpcc DB=mysql
+  - BENCH=tpcc DB=postgres
+  - BENCH=tpcc DB=tidb
+#  - BENCH=tpcc DB=memsql # FIXME: too many errors that exceeded travis log limit (4MB)
+# --- end tpcc with different databases ---
+# --- begin benchmarks using mysql ---
+  - BENCH=auctionmark DB=mysql 
+  # - BENCH=chbenchmark DB=mysql # FIXME: too many error https://travis-ci.org/benchhub/oltpbench/jobs/338773148
+  - BENCH=epinions DB=mysql
+  # - BENCH=hyadapt DB=mysql # TODO: no config
+  # - BENCH=jpab DB=mysql # TODO: no ddl
+  - BENCH=linkbench DB=mysql
+  - BENCH=noop DB=mysql
+  - BENCH=resourcestresser DB=mysql
+  - BENCH=seats DB=mysql
+  - BENCH=sibench DB=mysql
+  - BENCH=smallbank DB=mysql
+  - BENCH=tatp DB=mysql
+  # - BENCH=tpcc DB=mysql # dup
+  # - BENCH=tpch DB=mysql # TODO: require db gen
+  # - BENCH=tpcds DB=mysql # TODO: wonder how it worked, it didn't update plugin.xml when we were still using it
+  - BENCH=twitter DB=mysql # TODO: move traces location
+  - BENCH=voter DB=mysql
+  - BENCH=wikipedia DB=mysql
+  - BENCH=ycsb DB=mysql
+# --- end benchmarks using mysql --- 
+# --- begin other databases ---
+  - BENCH=ycsb DB=cassandra
+# --- end other databases ---
+# https://stackoverflow.com/questions/34377017/what-are-the-differences-between-the-before-install-script-travis-yml-opti
 before_install:
-  - sudo mount -o remount,size=50% /var/ramfs
-
+  - python3 --version
+  - sudo pip3 install pyyaml
+  - java -version
+  - docker version
+  - docker-compose version
+  # check if common ports are already in use, travis starts a bunch of DBMS by default ....
+  - sudo netstat -nlp | grep :3306
+  - sudo netstat -nlp | grep :5432
+  - sudo service mysql stop
+  - sudo service postgresql stop
+# https://docs.travis-ci.com/user/database-setup/#Multiple-Database-Builds
 install:
-  - echo $TRAVIS_JDK_VERSION
-  - if [ $TRAVIS_JDK_VERSION == oraclejdk8 ]; then
-      DB=mysql ;
-    else
-      DB=postgres ;
-    fi
-  - if [ $DB == mysql ]; then mysql -e "SELECT VERSION()";
-      mysql -e "CREATE DATABASE IF NOT EXISTS ${TEST}" ;
-      mysql -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY 'travis'; GRANT ALL ON *.* TO 'travis'@'localhost'";
-    elif [ $DB == postgres ]; then psql -c "SELECT VERSION()" -U travis;
-      psql -c "create database $TEST" -U postgres ;
-      psql -c "ALTER USER CURRENT_USER WITH PASSWORD 'travis'" -U travis;
-    fi
-
-# MySQL create user is probably one of the trust-container bugs for MySQL
-# Will probably need the following to set a password
-
-# We pass a password here as <password></password> sends no password in the protocol
-# rather than the blank one and therefore fails on MySQL.
-
+  - ant build
+  - ./docker/travis_start.sh
 before_script:
-  - if [ $DB == mysql ]; then
-      URLBASE=jdbc:mysql://localhost:3306 ;
-      DRIVER=com.mysql.jdbc.Driver ;
-      TYPE=mysql ;
-    elif [ $DB == postgres ]; then
-      URLBASE=jdbc:postgresql://localhost:5432 ;
-      DRIVER=org.postgresql.Driver ;
-      TYPE=postgres ;
-    fi
-
+  - ./config/config.py validate
 script:
-  - SCALEFACTOR=0.5
-  - if [ $TEST == junit ]; then
-      ant junit;
-    else
-      ant build;
-      config=config/sample_${TEST}_config.xml ;
-      sed -i
-           -e "/<dbtype>/c\<dbtype>${TYPE}</dbtype>"
-           -e "/<driver>/c\<driver>${DRIVER}</driver>"
-           -e "/<DBUrl>/c\<DBUrl>${URLBASE}/${TEST}</DBUrl>"
-           -e '/<username>/c\<username>travis</username>'
-           -e '/<password>/c\<password>travis</password>'
-           -e "/<scalefactor>/c\\<scalefactor>${SCALEFACTOR}</scalefactor>"
-           -e '/<terminals>/c\<terminals>3</terminals>'
-           -e '/<time>/c\<time>60</time>'
-           -e '/<isolation>/c\<isolation>TRANSACTION_READ_COMMITTED</isolation>'
-           "${config}";
-      ./oltpbenchmark --bench "${TEST}" --config "${config}"  --create true    --load true   --execute true ;
-    fi
-
-
-# With tests, scalefactor and terminals are just to reduce load on travis
-# isolation READ_COMMITTED as the TRANSACTIONAL default caused too many failures for now.
-
+  - ./config/config.py generate --bench ${BENCH} --db ${DB} --scalefactor 1
+  - ./oltpbenchmark --bench ${BENCH} --config config/generated_${BENCH}_${DB}_config.xml --create true --load true --execute true
 after_script:
-  - if [ $DB == mysql ]; then
-      mysql -e "DROP DATABASE IF EXISTS ${TEST}" ;
-    elif [ $DB == postgres ]; then
-      psql -c "DROP DATABASE IF EXISTS ${TEST}" -U postgres ;
-    fi
+  - ./docker/travis_stop.sh

config/.gitignore

@@ -0,0 +1 @@
+generated_*.xml

config/README.md

@@ -0,0 +1,130 @@
+# Configuration
+
+## How to
+
+### Use config.py
+
+````bash
+# print out benchmarks
+./config.py benchmarks
+# print out databases
+./config.py databases
+# validate yaml files
+./config.py validate
+# generate config using template
+./config.py --bench tpcc --db mysql
+````
+
+### Run a benchmark
+
+Use docker and docker-compose (not accurate, but easy to use)
+
+````bash
+# make sure you are in project root, NOT config folder
+export BENCH=tpcc
+export DB=mysql
+./config/config.py validate
+./config/config.py generate --bench ${BENCH} --db ${DB}
+# this will start database server and create database
+./docker/travis_start.sh
+./oltpbenchmark --bench ${BENCH} --config config/generated_${BENCH}_${DB}_config.xml --create true --load true --execute true
+````
+
+Use local database
+
+````bash
+export BENCH=tpcc
+export DB=mysql
+./config/config.py validate
+./config/config.py generate --bench ${BENCH} --db ${DB}
+# modify config/generated_${BENCH}_${DB}_config.xml username and password to match your local config
+# create database ${DB} use your local db shell
+./oltpbenchmark --bench ${BENCH} --config config/generated_${BENCH}_${DB}_config.xml --create true --load true --execute true
+````
+
+### Add a new benchmark config
+
+For instance we want to add TPCx-IoT http://www.tpc.org/tpcx-iot/default.asp
+
+- add the config template to `config/benchmarks/sample_tpcxiot_config.xml` use mysql as target database
+  - skip the rest if you don't use util scripts
+- update [benchmarks.yml](benchmarks.yml), it's a human readable catalog of all supported benchmarks
+  - benchmark are listed in alphabetical order
+- pick the primary name for the benchmark, it should be same as your java package name, i.e. TPC-C has package `com.oltpbenchmark.benchmarks.tpcc`, thus it's primary name is `tpcc`
+  - you can have alias, if you find the benchmark name too long. They will be used by util scripts when creating config and launching benchmarks, scripts will map them back to primary name before pass them to oltpbench
+- add optional information, there should be at least one item to make it a valid YAML.
+- run `config.py validate` to check if you have syntax error or there are name conflict (mainly due to alias)
+  - use `config.py --verbose validate` if something goes wrong (no output means good)
+- run `config.py benchmarks` you should see your benchmarks
+
+````yaml
+# (required) the primary name, should match what you use in java package name
+tpcxiot:
+  # (optional) displayname, just looks nicer
+  name: TPCx-IoT
+  # (optional) short description
+  description: a benchmark for IoT Gateway
+  # (optional) shorter names, common typos, only used by util scripts
+  alias:
+    - iot
+  # (optional) original homepage/paper link, could be dead ...
+  url: http://www.tpc.org/tpcx-iot/default.asp
+  # (optional) list of contributions in time order, use github id, prefer PR over commit hash for url
+  # not used by actual program, just a place to record you contribution
+  contributions:
+    - time: 2018-02-07
+      author: at15
+      url: https://github.com/oltpbenchmark/oltpbench/commit/cac6786e86869fd23ddb7813ced8357406c4aaef
+````
+
+### Add a new database config
+
+For instance we want to add MySQL
+
+- add the config template to `config/databases/sample_mysql_config.xml` use YCSB as target workload
+  - skip the rest if you don't use util scripts
+- update [databases.yml](databases.yml), it's a human readable catalog of some supported databases
+  - databases are grouped by type
+- pick the primary name, it will be used as `dbtype` in generated config
+  - you also need to update `com.oltpbenchmark.types.DatabaseType.java` to add your database i.e. `MYSQL("com.mysql.jdbc.Driver", true, false, true),`
+  - alias is same as benchmarks, used for util scripts only
+- jdbc url, default template is `DB_URL_TEMPLATE = 'jdbc:{dbms}://{host}:{port}/{db}'` in [config.py](config.py)
+  - {dbms} is the primary name
+  - {db} is the benchmark name, i.e. `tpcc`, `tpch`
+- many DBMS supports MySQL protocol, they should explicit specify their db url template, so `{dbms}` is not replaced
+- `username` and `password` should match what you specified for [docker](../docker)
+- `create_db` is the SQL to create database, note PostgreSQL doest not support `IF NOT EXISTS`
+- `shell` is the command to open a shell to database
+- `shell_exec` is the command to execute a single sql and exit, it is used for creating database on travis
+- `require_native_shell` is for database that don't ship db shell in their docker image, i.e. TiDB's docker image does not have mysql shell but you can use your local mysql shell to connect to it.
+
+````yaml
+mysql:
+  name: MySQL
+  alias:
+    - dolphin
+  driver: com.mysql.jdbc.Driver
+  port: 3306
+  username: root
+  password: oltpbenchpassword
+  create_db: "CREATE DATABASE IF NOT EXISTS {db}"
+  shell: "mysql -u {username} -p{password}"
+  shell_exec: "mysql -u {username} -p{password} -e \"{sql}\""
+````
+
+````yaml
+tidb:
+  name: TiDB
+  alias:
+    - Tidb
+  driver: com.mysql.jdbc.Driver
+  port: 4000
+  # NOTE: this is needed because tidb is mysql compataible
+  dburl: "jdbc:mysql://{host}:{port}/{db}"
+  username: root
+  password: ''
+  create_db: "CREATE DATABASE IF NOT EXISTS {db}"
+  shell: "mysql -u {username} -h 127.0.0.1 -P 4000"
+  shell_exec: "mysql -u {username} -h 127.0.0.1 -P 4000 -e \"{sql}\""
+  require_native_shell: true # tidb's docker container does not ship with mysql client
+````