From b06012d24dcb329a9d35bd9c629c0c1516f63cb1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 4 Jun 2014 12:31:01 +0200 Subject: [PATCH 01/75] updates sbt-assembly to a version that does exist --- build.sbt | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 426af60..02cb65d 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,6 @@ scalaVersion := "2.9.1" assemblySettings -jarName in assembly <<= (version) { v => "sqltap_" + v + ".jar" } +jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true diff --git a/project/plugins.sbt b/project/plugins.sbt index 9814732..9685838 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.5") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") From e0fa875a55165f88ffd652c7500db8dc2bbc0930 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:05:41 +0200 Subject: [PATCH 02/75] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e08c794..311cc1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ .DS_Store +/bin From 6d2b92cd5822a246ae0ceea79beb802d4303bc51 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:24:42 +0200 Subject: [PATCH 03/75] porting sqltap to Scala 2.10 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- src/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 02cb65d..90656dd 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.9.1" +scalaVersion := "2.10.4" assemblySettings diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/com/paulasmuth/sqltap/ExpirationJob.scala index 1d8f04b..354ec50 100644 --- a/src/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/com/paulasmuth/sqltap/ExpirationJob.scala @@ -45,7 +45,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] if (tuple._1 == primary_id) { val key = ctree.key(tuple._1, record_id.toString, tuple._2) handler.execute(worker, key) - keys = keys - tuple + keys = keys.filter(_ != tuple) } } } diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/com/paulasmuth/sqltap/ReplicationFeed.scala index 6f76440..52c988d 100644 --- a/src/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -42,7 +42,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { } else { val row = query.rows.last val position = row.last.toInt - val filename = row.first + val filename = row.head conn.start_binlog(filename, position) } From b1612602d2a58d7ee8ddbf872e5aee3190e3e517 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 27 Aug 2015 13:21:02 +0200 Subject: [PATCH 04/75] mysql: adds TIMESTAMP2 table column type to TableMapBinlogEvent --- src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala b/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala index 4c1f0c0..214bb45 100644 --- a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala +++ b/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala @@ -46,6 +46,7 @@ class TableMapBinlogEvent(data: Array[Byte], ts: Long, fmt: FormatDescriptionBin case 0x0c => 0 // 0x0c DATETIME case 0x0d => 0 // 0x0d YEAR case 0x0f => read_int(2) // 0x0f VARCHAR + case 0x11 => 0 // 0x11 TIMESTAMP2 case 0x12 => 0 // 0x12 DATETIME2 case 0xf6 => read_int(2) // 0xf6 NEWDECIMAL case 0xfc => read_int(1) // 0xfc BLOB From 9328b3fb2efb8ea0f00911250d86ce4ed83f4abc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:24:42 +0200 Subject: [PATCH 05/75] porting sqltap to Scala 2.10 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- src/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index cad9983..1223be8 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.9.1" +scalaVersion := "2.10.4" assemblySettings diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/com/paulasmuth/sqltap/ExpirationJob.scala index 1d8f04b..354ec50 100644 --- a/src/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/com/paulasmuth/sqltap/ExpirationJob.scala @@ -45,7 +45,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] if (tuple._1 == primary_id) { val key = ctree.key(tuple._1, record_id.toString, tuple._2) handler.execute(worker, key) - keys = keys - tuple + keys = keys.filter(_ != tuple) } } } diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/com/paulasmuth/sqltap/ReplicationFeed.scala index 6f76440..52c988d 100644 --- a/src/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -42,7 +42,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { } else { val row = query.rows.last val position = row.last.toInt - val filename = row.first + val filename = row.head conn.start_binlog(filename, position) } From 3b9b93bfbe95b3003727a27c185cc96ca962ba9b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 28 Oct 2014 15:44:42 +0100 Subject: [PATCH 06/75] some patches --- patches/sqltap-encoding-fix.diff | 62 +++++++++++++++++ patches/sqltap-log-slow-queries.diff | 100 +++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 patches/sqltap-encoding-fix.diff create mode 100644 patches/sqltap-log-slow-queries.diff diff --git a/patches/sqltap-encoding-fix.diff b/patches/sqltap-encoding-fix.diff new file mode 100644 index 0000000..5bb509c --- /dev/null +++ b/patches/sqltap-encoding-fix.diff @@ -0,0 +1,62 @@ +diff --git a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala +index 2145f56..cf17011 100644 +--- a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala ++++ b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -30,8 +30,10 @@ object ExpirationHandlerFactory { + case "noop" => + handler = new NoopExpirationHandler() + +- case "purge" => ++ case "purge" => { + handler = new PurgeExpirationHandler() ++ ReplicationFeed.start() ++ } + + case _ => + throw new ParseException("unknown expiration handler: " + name) +diff --git a/src/com/paulasmuth/sqltap/HTTPParser.scala b/src/com/paulasmuth/sqltap/HTTPParser.scala +index 22b27be..33cd26d 100644 +--- a/src/com/paulasmuth/sqltap/HTTPParser.scala ++++ b/src/com/paulasmuth/sqltap/HTTPParser.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -130,7 +130,7 @@ class HTTPParser { + } + + def uri_parts() : List[String] = { +- val uri = URLDecoder.decode(http_uri, "UTF-8") ++ val uri = URLDecoder.decode(http_uri) + var pos = uri.length + var cur = pos - 1 + var ret = new ListBuffer[String]() +diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala +index 357fc20..05c8921 100644 +--- a/src/com/paulasmuth/sqltap/SQLTap.scala ++++ b/src/com/paulasmuth/sqltap/SQLTap.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -108,7 +108,6 @@ object SQLTap{ + Manifest.load(new File(Config.get('config_base))) + RelationTrace.load(Manifest.resources) + ExpirationHandlerFactory.configure(Config.get('expiration_handler)) +- ReplicationFeed.start() + + val server = new Server(Config.get('threads).toInt) + server.run(Config.get('http_port).toInt) diff --git a/patches/sqltap-log-slow-queries.diff b/patches/sqltap-log-slow-queries.diff new file mode 100644 index 0000000..959f46b --- /dev/null +++ b/patches/sqltap-log-slow-queries.diff @@ -0,0 +1,100 @@ +diff -r -u sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala +--- sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala 2014-02-12 16:13:05.009387526 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala 2014-09-04 14:31:28.000000000 +0200 +@@ -1,10 +1,9 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of + // the License at: http://opensource.org/licenses/MIT +- + package com.paulasmuth.sqltap + + import java.nio.channels.{SocketChannel,SelectionKey} +@@ -23,6 +22,7 @@ + private val parser = new HTTPParser() + private var state = HTTP_STATE_INIT + private var last_event : SelectionKey = null ++ private var last_uri : String = "" // for debugging only + private var keepalive : Boolean = false + private var resp_buf : ByteBuffer = null + +@@ -190,6 +190,7 @@ + + idle_timer.cancel() + stime = System.nanoTime ++ last_uri = parser.http_uri + seq += 1 + + if (parser.http_version == "1.1") +@@ -302,8 +303,13 @@ + } + + def finish() : Unit = { +- Statistics.incr('http_request_time_mean, +- (System.nanoTime - stime) / 1000000.0) ++ val runtime_millis = (System.nanoTime - stime) / 1000000.0 ++ Statistics.incr('http_request_time_mean, runtime_millis) ++ ++ if (Config.has_key('log_slow_queries) && ++ runtime_millis >= Config.get('log_slow_queries).toInt) { ++ Logger.log("[HTTP] [Slow Query] (" + runtime_millis + "ms): " + last_uri) ++ } + + if (!keepalive) + return close() +diff -r -u sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala +--- sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala 2014-02-12 16:13:05.013389526 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala 2014-09-04 14:31:38.000000000 +0200 +@@ -1,10 +1,9 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of + // the License at: http://opensource.org/licenses/MIT +- + package com.paulasmuth.sqltap.mysql + + import com.paulasmuth.sqltap._ +@@ -44,8 +43,14 @@ + tok = System.nanoTime + qtime = tok - tik + +- Statistics.incr('sql_request_time_mean, qtime / 1000000.0) +- Logger.debug("Finished (" + (qtime / 1000000.0) + "ms): " + query) ++ val runtime_millis = qtime / 1000000.0 ++ Statistics.incr('sql_request_time_mean, runtime_millis) ++ Logger.debug("Finished (" + runtime_millis + "ms): " + query) ++ ++ if (Config.has_key('log_slow_queries) && ++ runtime_millis >= Config.get('log_slow_queries).toInt) { ++ Logger.log("[SQL] [Slow Query] (" + runtime_millis + "ms): " + query) ++ } + } + + def error(err: Throwable) : Unit = { +diff -r -u sqltap/src/com/paulasmuth/sqltap/SQLTap.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala +--- sqltap/src/com/paulasmuth/sqltap/SQLTap.scala 2014-02-25 17:15:47.049982142 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala 2014-09-04 14:35:51.000000000 +0200 +@@ -72,6 +71,9 @@ + else if (args(n) == "--disable-keepalive") + { Config.set('http_keepalive, "false"); n += 1 } + ++ else if (args(n) == "--log-slow-queries") ++ { Config.set('log_slow_queries, args(n+1)); n += 2 } ++ + else if ((args(n) == "-t") || (args(n) == "--threads")) + { Config.set('threads, args(n+1)); n += 2 } + +@@ -138,6 +139,7 @@ + println(" --memcache-queuelen max mysql queue size per worker ") + println(" --memcache-numconns max number of mysql connections per worker ") + println(" --memcache-mode replication mode (copy, shard) ") ++ println(" --log-slow-queries log all queries with a runtime > val in ms ") + println(" -h, --help you're reading it... ") + println(" -d, --debug debug mode ") + } From 4f292d789fc16e70c1c3035ba33cbc67f3b254b4 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Fri, 21 Nov 2014 17:59:23 +0100 Subject: [PATCH 07/75] Update to Scala 2.11.4 --- build.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1223be8..9250384 100644 --- a/build.sbt +++ b/build.sbt @@ -10,10 +10,12 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.10.4" +scalaVersion := "2.11.4" assemblySettings jarName in assembly := "sqltap_0.7.21.jar" fork in run := true + +libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" From b134c968f48b69df8d4433d44b58c3b627953884 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Fri, 21 Nov 2014 16:17:34 +0100 Subject: [PATCH 08/75] Parametrize memcache connection setting --- src/com/paulasmuth/sqltap/Config.scala | 2 ++ src/com/paulasmuth/sqltap/MemcacheConnection.scala | 5 +---- .../paulasmuth/sqltap/MemcacheConnectionPool.scala | 4 +++- src/com/paulasmuth/sqltap/SQLTap.scala | 12 ++++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/com/paulasmuth/sqltap/Config.scala index 94bab39..3cc2b13 100644 --- a/src/com/paulasmuth/sqltap/Config.scala +++ b/src/com/paulasmuth/sqltap/Config.scala @@ -24,6 +24,8 @@ object Config { 'memcache_mode -> "copy", 'memcache_queue_max_len -> "4096", 'memcache_max_connections -> "10", + 'memcache_host -> "127.0.0.1", + 'memcache_port -> "11211", 'threads -> "4", 'expiration_handler -> "purge", 'cache_backend -> "memcache" diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 8c3c5f4..bc3b01a 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -12,10 +12,7 @@ import java.nio.channels.{SocketChannel,SelectionKey} import java.nio.{ByteBuffer,ByteOrder} import java.net.{InetSocketAddress,ConnectException} -class MemcacheConnection(pool: MemcacheConnectionPool) extends TimeoutCallback { - - var hostname : String = "127.0.0.1" - var port : Int = 11211 +class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { private val MC_STATE_INIT = 0 private val MC_STATE_CONN = 1 diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala index 6c45c8f..12885f4 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala @@ -108,7 +108,9 @@ class MemcacheConnectionPool extends CacheBackend { } private def connect() : Unit = { - val conn = new MemcacheConnection(this) + val port: Int = Config.get('memcache_port).toInt + val host: String = Config.get('memcache_host) + val conn = new MemcacheConnection(this, host, port) conn.connect() connections += conn diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index 8f6477b..67aadf0 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -56,9 +56,6 @@ object SQLTap{ else if (args(n) == "--cache-backend") { Config.set('cache_backend, args(n+1)); n += 2 } - else if (args(n) == "--memcache-hosts") - { Config.set('memcache_hosts, args(n+1)); n += 2 } - else if (args(n) == "--memcache-mode") { Config.set('memcache_mode, args(n+1)); n += 2 } @@ -74,6 +71,12 @@ object SQLTap{ else if (args(n) == "--log-slow-queries") { Config.set('log_slow_queries, args(n+1)); n += 2 } + else if (args(n) == "--memcache-host") + { Config.set('memcache_host, args(n+1)); n += 2 } + + else if (args(n) == "--memcache-port") + { Config.set('memcache_port, args(n+1)); n += 2 } + else if ((args(n) == "-t") || (args(n) == "--threads")) { Config.set('threads, args(n+1)); n += 2 } @@ -135,7 +138,8 @@ object SQLTap{ println(" --mysql-numconns max number of mysql connections per worker ") println(" --expiration-handler expiration handler (noop, purge, refresh) ") println(" --cache-backend cache backend (memcache) ") - println(" --memcache-hosts comma-seperated memcache servers (host:port) ") + println(" --memcache-host memcache server host ") + println(" --memcache-port memcache server port ") println(" --memcache-queuelen max mysql queue size per worker ") println(" --memcache-numconns max number of mysql connections per worker ") println(" --memcache-mode replication mode (copy, shard) ") From 9f9fa5cfb4dad7b067fd1e3434ba30a77fe65c07 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Mon, 2 Feb 2015 17:33:30 +0100 Subject: [PATCH 09/75] Add TTL support per ctree --- src/com/paulasmuth/sqltap/CTree.scala | 1 + src/com/paulasmuth/sqltap/CTreeCache.scala | 2 +- src/com/paulasmuth/sqltap/CacheStoreRequest.scala | 2 +- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CTree.scala b/src/com/paulasmuth/sqltap/CTree.scala index 2d2f941..8d4a626 100644 --- a/src/com/paulasmuth/sqltap/CTree.scala +++ b/src/com/paulasmuth/sqltap/CTree.scala @@ -12,6 +12,7 @@ class CTree(doc: xml.Node) { val name : String = elem.attr("name", true) val query : String = elem.attr("query", true) + val expire : Int = elem.attr("expire", false, "300").toInt val allow_conditions : Boolean = elem.attr("allow_conditions", false, "true").equals("true") diff --git a/src/com/paulasmuth/sqltap/CTreeCache.scala b/src/com/paulasmuth/sqltap/CTreeCache.scala index 4287492..234f58a 100644 --- a/src/com/paulasmuth/sqltap/CTreeCache.scala +++ b/src/com/paulasmuth/sqltap/CTreeCache.scala @@ -17,7 +17,7 @@ object CTreeCache { CTreeMarshal.serialize(ctree_buf, ctree.stack.head, ins) - val request = new CacheStoreRequest(key, buf) + val request = new CacheStoreRequest(key, buf, ctree.expire) request.worker = worker worker.cache.enqueue(request) diff --git a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala b/src/com/paulasmuth/sqltap/CacheStoreRequest.scala index e39c009..578300c 100644 --- a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala +++ b/src/com/paulasmuth/sqltap/CacheStoreRequest.scala @@ -9,7 +9,7 @@ package com.paulasmuth.sqltap import scala.collection.mutable.{ListBuffer} -class CacheStoreRequest(_key: String, _buf: ElasticBuffer) extends CacheRequest { +class CacheStoreRequest(_key: String, _buf: ElasticBuffer, val expire: Int) extends CacheRequest { val key : String = _key buffer = _buf diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index bc3b01a..8ab4ab9 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -98,7 +98,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put(32.toByte) write_buf.put(48.toByte) write_buf.put(32.toByte) - write_buf.put(48.toByte) + write_buf.put(request.expire.toString.getBytes("UTF-8")) write_buf.put(32.toByte) write_buf.put(len.toString.getBytes("UTF-8")) write_buf.put(13.toByte) From bf870f58e34467d1481fe09190a9d16856d83e04 Mon Sep 17 00:00:00 2001 From: Tadas Sce Date: Mon, 4 May 2015 15:47:15 +0300 Subject: [PATCH 10/75] Escape backslashes when writing json --- src/com/paulasmuth/sqltap/JSONWriter.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/com/paulasmuth/sqltap/JSONWriter.scala b/src/com/paulasmuth/sqltap/JSONWriter.scala index 94b4502..2902650 100644 --- a/src/com/paulasmuth/sqltap/JSONWriter.scala +++ b/src/com/paulasmuth/sqltap/JSONWriter.scala @@ -81,10 +81,12 @@ class JSONWriter(buf: WrappedBuffer) { val b = byte & 0x000000ff if (b == 0xA) { - buf.write(Array(0x5C.toByte, 0x6E.toByte)) + buf.write(Array(0x5C.toByte, 0x6E.toByte)) // \n } else if (b == 0x22) { - buf.write(Array(0x5C.toByte, 0x22.toByte)) - } else if ((b == 0) || ((b >= 0x20) && (b != 0x5C))) { + buf.write(Array(0x5C.toByte, 0x22.toByte)) // \" + } else if (b == 0x5C) { + buf.write(Array(0x5C.toByte, 0x5C.toByte)) // \\ + } else if ((b == 0) || ((b >= 0x20))) { buf.write(byte) } } From 2ff2a4c2f64707292479e9a561ff25365346ed28 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:05:41 +0200 Subject: [PATCH 11/75] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e08c794..311cc1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ .DS_Store +/bin From 890163e61b3eaa28aff3a400beee850984897c54 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 4 Jun 2014 12:31:01 +0200 Subject: [PATCH 12/75] updates sbt-assembly to a version that does exist --- build.sbt | 2 +- project/plugins.sbt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 project/plugins.sbt diff --git a/build.sbt b/build.sbt index 9250384..fcd6c62 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ scalaVersion := "2.11.4" assemblySettings -jarName in assembly := "sqltap_0.7.21.jar" +jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..9685838 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,3 @@ +resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) + +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") From a00e4a6b1803455054b5f45c458e386ccab29c96 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:03:55 +0200 Subject: [PATCH 13/75] fixes memcache_connections_open / sql_connections_open statistics bug --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 8ab4ab9..407615d 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -226,7 +226,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pool.close(this) sock.close() - Statistics.decr('sql_connections_open) + Statistics.decr('memcache_connections_open) } def timeout() : Unit = { From 7a00108c881a9e71491e4a2de09e40bd05371d00 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:04:37 +0200 Subject: [PATCH 14/75] symbolize literals to make the source code a little more readable --- .../sqltap/MemcacheConnection.scala | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 407615d..ef78fa9 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -14,6 +14,10 @@ import java.net.{InetSocketAddress,ConnectException} class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { + private val CR = 13 + private val LF = 10 + private val SP = 32 + private val MC_STATE_INIT = 0 private val MC_STATE_CONN = 1 private val MC_STATE_IDLE = 2 @@ -66,12 +70,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put("get".getBytes) for (key <- keys) { - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) } - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_MGET @@ -93,19 +97,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("set".getBytes) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(48.toByte) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(request.expire.toString.getBytes("UTF-8")) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(len.toString.getBytes("UTF-8")) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.put(buf) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_SET @@ -120,10 +124,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("delete".getBytes) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_DELETE @@ -171,7 +175,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pos = cur } else { - if (read_buf.get(cur) == 10) { + if (read_buf.get(cur) == LF) { next(new String(read_buf.array, pos, cur - 1 - pos, "UTF-8")) pos = cur + 1 } From 36dc9c3b92110a662fd1aa5db6f30d25c892018f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:07:00 +0200 Subject: [PATCH 15/75] helps 48.toByte to be more readable, it's a '0' literal ;-) --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index ef78fa9..7c05184 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -100,7 +100,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) write_buf.put(SP.toByte) - write_buf.put(48.toByte) + write_buf.put('0'.toByte) write_buf.put(SP.toByte) write_buf.put(request.expire.toString.getBytes("UTF-8")) write_buf.put(SP.toByte) From 610f3a18b6f4f45d30356936edca093ff831b301 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 2 Oct 2015 17:16:56 +0200 Subject: [PATCH 16/75] Fixes bug with too large memcached responses and its follow-up resource leakage --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 7c05184..08f1643 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -27,8 +27,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private val MC_STATE_READ = 7 private val MC_STATE_CLOSE = 8 - private val MC_WRITE_BUF_LEN = 65535 - private val MC_READ_BUF_LEN = (65535 * 8) + private val MC_WRITE_BUF_LEN = (65535 * 3) + private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) private var state = MC_STATE_INIT private var last_event : SelectionKey = null @@ -87,6 +87,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : val len = buf.position request.ready() + + if (len > MC_WRITE_BUF_LEN - 512) { + // some hacky safe margin + // minus 512 because of the first command line + return; + } + buf.position(0) buf.limit(len) From b5ba51fd542f2874ffdd8e0b4e7e8729a500c902 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 2 Oct 2015 18:16:04 +0200 Subject: [PATCH 17/75] some personally helpful documentation --- .../sqltap/MemcacheConnection.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 08f1643..d0a6974 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -40,8 +40,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : sock.configureBlocking(false) private var timer = TimeoutScheduler.schedule(1000, this) + private var requests : List[CacheRequest] = null + + // buffer for the current value to be received private var cur_buf : ElasticBuffer = null + + // length in bytes of the currently received value private var cur_len = 0 def connect() : Unit = { @@ -102,6 +107,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : timer.start() + // "set $key 0 $expiry $len\r\n$buf\r\n" write_buf.clear write_buf.put("set".getBytes) write_buf.put(SP.toByte) @@ -155,6 +161,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : idle(event) } + /** + * @brief Callback, invoked when underlyingsocket is non-blocking readable. + * + * Processes any incoming data, i.e. the response from the underlying + * memcached server. + */ def read(event: SelectionKey) : Unit = { val chunk = sock.read(read_buf) @@ -201,6 +213,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } + /** + * @brief Callback, invoked when underlying connection is non-blocking + * writable. + * + * When all data has been flushed out to the memcached server, + * we will stop watching for WRITE events and switch back to READ. + */ def write(event: SelectionKey) : Unit = { try { sock.write(write_buf) @@ -287,6 +306,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return idle(last_event) } + // expect ["VALUE", key, 0, dataLength] if (parts.length != 4) { throw new ExecutionException("[Memcache] protocol error: " + cmd) } From 60c309d72545ed5bd2520fd46aff5b01f576315c Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 10:20:58 +0200 Subject: [PATCH 18/75] documentation updates, def rearrangements, and minor code cleanups No logic has been changed. --- .../sqltap/MemcacheConnection.scala | 198 +++++++++--------- 1 file changed, 103 insertions(+), 95 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index d0a6974..3809442 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -12,20 +12,27 @@ import java.nio.channels.{SocketChannel,SelectionKey} import java.nio.{ByteBuffer,ByteOrder} import java.net.{InetSocketAddress,ConnectException} +/** + * @param pool pool that owns this connection + * @param hostname memcached-server's hostname + * @param port memcached-server's port number + */ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { - private val CR = 13 - private val LF = 10 - private val SP = 32 - - private val MC_STATE_INIT = 0 - private val MC_STATE_CONN = 1 - private val MC_STATE_IDLE = 2 - private val MC_STATE_CMD_DELETE = 4 - private val MC_STATE_CMD_SET = 5 - private val MC_STATE_CMD_MGET = 6 - private val MC_STATE_READ = 7 - private val MC_STATE_CLOSE = 8 + private val CR = 13.toByte + private val LF = 10.toByte + private val SP = 32.toByte + + // states: + // Uninitialized, Connecting, Idle, Cmd{Delete,Set,MGet}, Reading, Closed. + private val MC_STATE_INIT = 0 // just created, no connect() invoked yet (idle) + private val MC_STATE_CONN = 1 // just connected (idle) + private val MC_STATE_IDLE = 2 // idle (after a command) + private val MC_STATE_CMD_DELETE = 4 // + private val MC_STATE_CMD_SET = 5 // + private val MC_STATE_CMD_MGET = 6 // executing a multi-key GET + private val MC_STATE_READ = 7 // + private val MC_STATE_CLOSE = 8 // private val MC_WRITE_BUF_LEN = (65535 * 3) private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) @@ -49,6 +56,11 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // length in bytes of the currently received value private var cur_len = 0 + /** + * @brief Asynchronously establishes connection to the Memcached server. + * + * @see ready(event: SelectionKey) + */ def connect() : Unit = { Statistics.incr('memcache_connections_open) @@ -63,6 +75,42 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : .attach(this) } + /** + * @brief Callback, invoked upon non-blocking connect() completion. + * + * Moves MemcacheConnection state from MC_STATE_CONN into MC_STATE_IDLE state. + */ + def ready(event: SelectionKey) : Unit = { + try { + sock.finishConnect + } catch { + case e: ConnectException => { + Logger.error("[Memcache] connection failed: " + e.toString, false) + return close(e) + } + } + + idle(event) + } + + /** + * @brief Puts connection into ready state and then back into the idle pool. + */ + private def idle(event: SelectionKey) : Unit = { + timer.cancel() + state = MC_STATE_IDLE + event.interestOps(0) + last_event = event + requests = null + pool.ready(this) + } + + /** + * @brief Retrieves values for multiple keys. + * + * @param keys list of keys to retrieve + * @param _requests list of CacheRequest objects to store the values to + */ def execute_mget(keys: List[String], _requests: List[CacheRequest]) : Unit = { requests = _requests @@ -75,12 +123,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put("get".getBytes) for (key <- keys) { - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) } - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_MGET @@ -110,19 +158,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // "set $key 0 $expiry $len\r\n$buf\r\n" write_buf.clear write_buf.put("set".getBytes) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put('0'.toByte) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(request.expire.toString.getBytes("UTF-8")) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(len.toString.getBytes("UTF-8")) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.put(buf) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_SET @@ -137,32 +185,18 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("delete".getBytes) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_DELETE last_event.interestOps(SelectionKey.OP_WRITE) } - - def ready(event: SelectionKey) : Unit = { - try { - sock.finishConnect - } catch { - case e: ConnectException => { - Logger.error("[Memcache] connection failed: " + e.toString, false) - return close(e) - } - } - - idle(event) - } - /** - * @brief Callback, invoked when underlyingsocket is non-blocking readable. + * @brief Callback, invoked when underlying socket is non-blocking readable. * * Processes any incoming data, i.e. the response from the underlying * memcached server. @@ -195,7 +229,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pos = cur } else { if (read_buf.get(cur) == LF) { - next(new String(read_buf.array, pos, cur - 1 - pos, "UTF-8")) + val headline = new String(read_buf.array, pos, cur - 1 - pos, "UTF-8") + next(headline) pos = cur + 1 } @@ -264,37 +299,29 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : close() } + /** @brief Retrieves the corresponding CacheRequest to the given @p key. + * + * @return never null but the CacheRequest object. + */ + private def get_request_by_key(key: String) : CacheRequest = { + for (req <- requests) { + if (req.buffer == null && req.key == key) { + return req + } + } + throw new ExecutionException("[Memcache] invalid response key: " + key) + } + private def next(cmd: String) : Unit = { state match { - - case MC_STATE_CMD_DELETE => { - cmd match { - - case "DELETED" => { - idle(last_event) - } - - case "NOT_FOUND" => { - idle(last_event) - } - - } + case MC_STATE_CMD_DELETE => cmd match { + case "DELETED" => idle(last_event) + case "NOT_FOUND" => idle(last_event) } - - case MC_STATE_CMD_SET => { - cmd match { - - case "STORED" => { - idle(last_event) - } - - case "NOT_STORED" => { - idle(last_event) - } - - } + case MC_STATE_CMD_SET => cmd match { + case "STORED" => idle(last_event) + case "NOT_STORED" => idle(last_event) } - case MC_STATE_CMD_MGET => { val parts = cmd.split(" ") @@ -311,36 +338,17 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : throw new ExecutionException("[Memcache] protocol error: " + cmd) } - for (req <- requests) { - if (req.buffer == null && req.key == parts(1)) { - val buf = new ElasticBuffer(65535 * 8) - req.buffer = buf + val req = get_request_by_key(parts(1)) + cur_buf = new ElasticBuffer(65535 * 8) + cur_len = parts(3).toInt + req.buffer = cur_buf - cur_buf = buf - cur_len = parts(3).toInt - - state = MC_STATE_READ - return - } - } + state = MC_STATE_READ } - case _ => { - throw new ExecutionException( - "unexpected token " + cmd + " (" + state.toString + ")") + throw new ExecutionException("unexpected token " + cmd + + " (" + state.toString + ")") } - } } - - private def idle(event: SelectionKey) : Unit = { - timer.cancel() - state = MC_STATE_IDLE - event.interestOps(0) - last_event = event - requests = null - pool.ready(this) - } - - } From e2e323b9249cd3cc668142c85e6c5d1ec2c5f0a1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 10:34:43 +0200 Subject: [PATCH 19/75] code readability improvements --- .../sqltap/CacheBackendFactory.scala | 15 +++------------ .../paulasmuth/sqltap/MemcacheConnection.scala | 6 ++++++ .../sqltap/MemcacheConnectionPool.scala | 18 +++--------------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala b/src/com/paulasmuth/sqltap/CacheBackendFactory.scala index ec883b6..c5fadc0 100644 --- a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala +++ b/src/com/paulasmuth/sqltap/CacheBackendFactory.scala @@ -8,25 +8,16 @@ package com.paulasmuth.sqltap object CacheBackendFactory { - def get(worker: Worker) : CacheBackend = { val name = Config.get('cache_backend) val backend = name match { - - case "memcache" => - new MemcacheConnectionPool() - - case "noop" => - new NoopCacheBackend() - - case _ => - throw new ParseException("unknown cache backend: " + name) - + case "memcache" => new MemcacheConnectionPool() + case "noop" => new NoopCacheBackend() + case _ => throw new ParseException("unknown cache backend: " + name) } backend.loop = worker.loop backend } - } diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 3809442..4684bcb 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -112,6 +112,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : * @param _requests list of CacheRequest objects to store the values to */ def execute_mget(keys: List[String], _requests: List[CacheRequest]) : Unit = { + Logger.debug("[Memcache] mget: " + keys.mkString(", ")) + requests = _requests if (state != MC_STATE_IDLE) @@ -136,6 +138,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } def execute_set(key: String, request: CacheStoreRequest) : Unit = { + Logger.debug("[Memcache] store: " + key) + val buf = request.buffer.buffer val len = buf.position @@ -178,6 +182,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } def execute_delete(key: String) : Unit = { + Logger.debug("[Memcache] delete: " + key) + if (state != MC_STATE_IDLE) throw new ExecutionException("memcache connection busy") diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala index 12885f4..0f42f4e 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala @@ -28,8 +28,7 @@ class MemcacheConnectionPool extends CacheBackend { if (queue.length >= max_queue_len) { requests.foreach(_.ready()) - Logger.exception( - new TemporaryException("memcache queue is full"), false) + Logger.exception(new TemporaryException("memcache queue is full"), false) return } @@ -91,7 +90,6 @@ class MemcacheConnectionPool extends CacheBackend { batch += req } - Logger.debug("[Memcache] mget: " + keys.mkString(", ")) conn.execute_mget(keys.toList, batch.toList) execute_next() @@ -118,18 +116,8 @@ class MemcacheConnectionPool extends CacheBackend { private def execute(connection: MemcacheConnection, req: CacheRequest) = { req match { - - case set: CacheStoreRequest => { - Logger.debug("[Memcache] store: " + req.key) - connection.execute_set(req.key, set) - } - - case purge: CachePurgeRequest => { - Logger.debug("[Memcache] delete: " + req.key) - connection.execute_delete(purge.key) - } - + case set: CacheStoreRequest => connection.execute_set(req.key, set) + case purge: CachePurgeRequest => connection.execute_delete(purge.key) } } - } From ce21b89ac6d7708f529dd4e65774396b636d63ac Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 13:23:43 +0200 Subject: [PATCH 20/75] documentation --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 4684bcb..2324ea6 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -31,8 +31,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private val MC_STATE_CMD_DELETE = 4 // private val MC_STATE_CMD_SET = 5 // private val MC_STATE_CMD_MGET = 6 // executing a multi-key GET - private val MC_STATE_READ = 7 // - private val MC_STATE_CLOSE = 8 // + private val MC_STATE_READ = 7 // reading an execute_mget value chunk + private val MC_STATE_CLOSE = 8 // connection closed private val MC_WRITE_BUF_LEN = (65535 * 3) private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) @@ -318,6 +318,16 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : throw new ExecutionException("[Memcache] invalid response key: " + key) } + /** + * @brief Processes a command response. + * + * @param the first line of the response + * + * Usually commands have a response of only one command, thus, they'll + * directly transition the connection to the idle-state. + * + * Other commands (such as GET) may require reading more data. + */ private def next(cmd: String) : Unit = { state match { case MC_STATE_CMD_DELETE => cmd match { From 6f765c8bb1b6bca72b8f012dde6cb1bc1b56c87a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 13:57:38 +0200 Subject: [PATCH 21/75] code cleanup: keep single-statement for-loops more consistent with what has been found in the code base already --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 2324ea6..866edfd 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -283,9 +283,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : try { if (requests != null) { - for (req <- requests) { - req.ready() - } + requests.foreach(_.ready()) } } catch { case e: Exception => { @@ -342,10 +340,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : val parts = cmd.split(" ") if (parts.length == 1 && parts.head == "END") { - for (req <- requests) { - req.ready() - } - + requests.foreach(_.ready()) return idle(last_event) } From fad16b6672e885979197c7c808ca6071df0f189e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:23:33 +0200 Subject: [PATCH 22/75] more documentationary --- .../sqltap/MemcacheConnection.scala | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 866edfd..1ec8c1b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -18,7 +18,6 @@ import java.net.{InetSocketAddress,ConnectException} * @param port memcached-server's port number */ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { - private val CR = 13.toByte private val LF = 10.toByte private val SP = 32.toByte @@ -53,7 +52,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // buffer for the current value to be received private var cur_buf : ElasticBuffer = null - // length in bytes of the currently received value + // length in bytes of the currently received value still to be processed. private var cur_len = 0 /** @@ -216,18 +215,20 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return } - var cur = 0 - var pos = 0 + var cur = 0 //!< the offset in read_buf currently being parsed + var pos = 0 //!< offset of the first byte of the currently processed chunk (message line or value payload) while (cur < read_buf.position) { if (state == MC_STATE_READ) { + // process response body chunk, from pos to min(maxpos, pos + cur_len) cur = math.min(read_buf.position, pos + cur_len) cur_len -= cur - pos cur_buf.write(read_buf.array, pos, cur - pos) + // GET-value response chunk fully consumed? if (cur_len == 0) { - cur += 2 + cur += 2 // skip "\r\n" cur_buf.buffer.flip() state = MC_STATE_CMD_MGET } @@ -277,6 +278,11 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } + /** + * @brief Closes this connection and notifies the owning pool about the close. + * + * @param err The exception that potentially caused the close. + */ def close(err: Throwable = null) : Unit = { if (state == MC_STATE_CLOSE) return @@ -298,6 +304,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Statistics.decr('memcache_connections_open) } + /** @brief Callback, invoked upon I/O completion timeout. + * + * Closes the memcache connection. + */ def timeout() : Unit = { Logger.error("[Memcache] connection timed out...", false) close() @@ -350,8 +360,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } val req = get_request_by_key(parts(1)) - cur_buf = new ElasticBuffer(65535 * 8) cur_len = parts(3).toInt + cur_buf = new ElasticBuffer(65535 * 8) // FIXME why not of size cur_len? req.buffer = cur_buf state = MC_STATE_READ From c4a5c83974b8ddae249aaddd0720127173c9a31f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:25:16 +0200 Subject: [PATCH 23/75] minor code-refactor --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 1ec8c1b..81d744b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -318,12 +318,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : * @return never null but the CacheRequest object. */ private def get_request_by_key(key: String) : CacheRequest = { - for (req <- requests) { - if (req.buffer == null && req.key == key) { - return req - } + requests.find(r => r.key == key && r.buffer == null) match { + case Some(r) => r + case None => throw new ExecutionException("[Memcache] invalid response key: " + key) } - throw new ExecutionException("[Memcache] invalid response key: " + key) } /** From 793037ff4f1c3ea3ccb4059635a41f917ceef8cc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:57:26 +0200 Subject: [PATCH 24/75] cleanup: remove unnecessary SelectionKey passing, as it is always the same. --- .../sqltap/MemcacheConnection.scala | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 81d744b..964a27d 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -69,9 +69,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : timer.start() - sock - .register(pool.loop, SelectionKey.OP_CONNECT) - .attach(this) + last_event = sock.register(pool.loop, SelectionKey.OP_CONNECT) + last_event.attach(this) } /** @@ -89,17 +88,16 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } - idle(event) + idle() } /** * @brief Puts connection into ready state and then back into the idle pool. */ - private def idle(event: SelectionKey) : Unit = { + private def idle() : Unit = { timer.cancel() state = MC_STATE_IDLE - event.interestOps(0) - last_event = event + last_event.interestOps(0) requests = null pool.ready(this) } @@ -274,7 +272,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : if (write_buf.remaining == 0) { write_buf.clear - event.interestOps(SelectionKey.OP_READ) + last_event.interestOps(SelectionKey.OP_READ) } } @@ -337,19 +335,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private def next(cmd: String) : Unit = { state match { case MC_STATE_CMD_DELETE => cmd match { - case "DELETED" => idle(last_event) - case "NOT_FOUND" => idle(last_event) + case "DELETED" => idle() + case "NOT_FOUND" => idle() } case MC_STATE_CMD_SET => cmd match { - case "STORED" => idle(last_event) - case "NOT_STORED" => idle(last_event) + case "STORED" => idle() + case "NOT_STORED" => idle() } case MC_STATE_CMD_MGET => { val parts = cmd.split(" ") if (parts.length == 1 && parts.head == "END") { requests.foreach(_.ready()) - return idle(last_event) + return idle() } // expect ["VALUE", key, 0, dataLength] From cbec21b845f4a58c52cb583e9d17d8daf5162a7e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 21:34:03 +0200 Subject: [PATCH 25/75] bugfix for the infamous Memcached protocol error --- .../sqltap/MemcacheConnection.scala | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 964a27d..c43829b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -213,44 +213,45 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return } - var cur = 0 //!< the offset in read_buf currently being parsed - var pos = 0 //!< offset of the first byte of the currently processed chunk (message line or value payload) - - while (cur < read_buf.position) { + while (read_buf.position > 0) { if (state == MC_STATE_READ) { // process response body chunk, from pos to min(maxpos, pos + cur_len) - cur = math.min(read_buf.position, pos + cur_len) + val cur_chunk_len = math.min(read_buf.position, cur_len) - cur_len -= cur - pos - cur_buf.write(read_buf.array, pos, cur - pos) + cur_len -= cur_chunk_len + cur_buf.write(read_buf.array, 0, cur_chunk_len) // GET-value response chunk fully consumed? if (cur_len == 0) { - cur += 2 // skip "\r\n" + cur_buf.retrieve.limit(cur_buf.retrieve.limit() - 2) cur_buf.buffer.flip() state = MC_STATE_CMD_MGET } - pos = cur + read_buf.limit(read_buf.position) + read_buf.position(cur_chunk_len) + read_buf.compact() } else { - if (read_buf.get(cur) == LF) { - val headline = new String(read_buf.array, pos, cur - 1 - pos, "UTF-8") - next(headline) - pos = cur + 1 + var found = false; + var i = 0; + + while (!found && i < read_buf.position) { + if (read_buf.get(i) == LF) { + val headline = new String(read_buf.array, 0, i - 1, "UTF-8") + read_buf.limit(read_buf.position) + read_buf.position(i + 1) + read_buf.compact() + next(headline) + found = true; + } + i = i + 1; } - cur += 1 + if (!found) { + return; + } } } - - if (cur < read_buf.position) { - println("READ REMAINING") - read_buf.limit(read_buf.position) - read_buf.position(cur) - read_buf.compact() - } else { - read_buf.clear() - } } /** @@ -356,7 +357,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } val req = get_request_by_key(parts(1)) - cur_len = parts(3).toInt + cur_len = parts(3).toInt + 2 cur_buf = new ElasticBuffer(65535 * 8) // FIXME why not of size cur_len? req.buffer = cur_buf From e77af946cc5e8b3c70235aff1c5867b110a489c8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 23:28:21 +0200 Subject: [PATCH 26/75] mute record-not-found log messages --- src/com/paulasmuth/sqltap/Worker.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Worker.scala b/src/com/paulasmuth/sqltap/Worker.scala index 3b7a97b..40ae809 100644 --- a/src/com/paulasmuth/sqltap/Worker.scala +++ b/src/com/paulasmuth/sqltap/Worker.scala @@ -80,6 +80,11 @@ class Worker() extends Thread { conn.write(event) } catch { + case e: NotFoundException => { + // but do not log it, as it's a client side error we do not care + // about in the server side log + conn.close(e) + } case e: Exception => { Logger.error("[SQL] exception: " + e.toString, false) Logger.exception(e, false) From a45101a5a38829926d2cdb0ba0d93d14c610e2b3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 23:28:40 +0200 Subject: [PATCH 27/75] version bump 0.8.0 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/SQLTap.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index fcd6c62..bb30c99 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.7.21" +version := "0.8.0" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index 67aadf0..e6152b2 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -19,7 +19,7 @@ import java.io.File object SQLTap{ - val VERSION = "v0.7.21" + val VERSION = "v0.8.0" def main(args: Array[String]) : Unit = { var n = 0 From 80d6887f5f2e38e113b1c15c12b1ce455a4a3580 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 02:07:53 +0200 Subject: [PATCH 28/75] dead code cleanup --- src/com/paulasmuth/sqltap/StubCache.scala | 45 ----------------------- 1 file changed, 45 deletions(-) delete mode 100644 src/com/paulasmuth/sqltap/StubCache.scala diff --git a/src/com/paulasmuth/sqltap/StubCache.scala b/src/com/paulasmuth/sqltap/StubCache.scala deleted file mode 100644 index 7fdb2cc..0000000 --- a/src/com/paulasmuth/sqltap/StubCache.scala +++ /dev/null @@ -1,45 +0,0 @@ -// This file is part of the "SQLTap" project -// (c) 2011-2013 Paul Asmuth -// -// Licensed under the MIT License (the "License"); you may not use this -// file except in compliance with the License. You may obtain a copy of -// the License at: http://opensource.org/licenses/MIT - -package com.paulasmuth.sqltap - -import scala.collection.mutable.{HashMap} - -// STUB! -class StubCache extends CacheBackend { - - val stubcache = new HashMap[String,ElasticBuffer]() - - def connect() : Unit = () - - def execute(requests: List[CacheRequest]) = { - for (req <- requests) { - req match { - case get: CacheGetRequest => { - Logger.debug("[CACHE] retrieve: " + req.key) - stubcache.get(req.key) match { - case Some(buf: ElasticBuffer) => { - get.buffer = buf.clone() - } - case None => () - } - } - case set: CacheStoreRequest => { - Logger.debug("[CACHE] store: " + req.key) - stubcache.put(req.key, set.buffer) - } - case purge: CachePurgeRequest => { - Logger.debug("[CACHE] purge: " + req.key) - stubcache.remove(req.key) - } - } - - req.ready() - } - } - -} From 504f1ae80ffdcd54cec890ad7f5b857a7997df84 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:21:41 +0200 Subject: [PATCH 29/75] Improve logger api --- src/com/paulasmuth/sqltap/Logger.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Logger.scala b/src/com/paulasmuth/sqltap/Logger.scala index 0cf4469..6b003a6 100644 --- a/src/com/paulasmuth/sqltap/Logger.scala +++ b/src/com/paulasmuth/sqltap/Logger.scala @@ -19,6 +19,11 @@ object Logger { println("[" + df.format(new Date()) + "] " + msg) } + def fatal(msg: String) : Unit = { + log("[FATAL] " + msg) + System.exit(1) + } + def error(msg: String, fatal: Boolean) : Unit = { log("[ERROR] " + msg) @@ -26,6 +31,14 @@ object Logger { System.exit(1) } + def notice(msg: String) : Unit = { + log("[NOTICE] " + msg) + } + + def info(msg: String) : Unit = { + log("[INFO] " + msg) + } + def debug(msg: String) : Unit = { if (Config.debug) log("[DEBUG] " + msg) From f049da75a3c90cbe8e7d5f331a759a054617e887 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:21:52 +0200 Subject: [PATCH 30/75] fixes compiler warning --- src/com/paulasmuth/sqltap/GZIPTranscoder.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala b/src/com/paulasmuth/sqltap/GZIPTranscoder.scala index 0be3aa3..50a56a9 100644 --- a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala +++ b/src/com/paulasmuth/sqltap/GZIPTranscoder.scala @@ -28,7 +28,7 @@ class GZIPTranscoder(buffer: ElasticBuffer) { buf.put(target.toByteArray()) } - def decode() : Unit = try { + def decode() : Unit = { val buf = buffer.retrieve() val source = new ByteArrayInputStream(buf.array.clone()) val gzip = new GZIPInputStream(source) @@ -50,7 +50,6 @@ class GZIPTranscoder(buffer: ElasticBuffer) { gzip.close source.close } - } From d234cfc4e2c12e6ef4ae8a1550716bc5d7bb66d3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:22:29 +0200 Subject: [PATCH 31/75] code cleanup --- src/com/paulasmuth/sqltap/CTreeIndex.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CTreeIndex.scala b/src/com/paulasmuth/sqltap/CTreeIndex.scala index e528950..1017313 100644 --- a/src/com/paulasmuth/sqltap/CTreeIndex.scala +++ b/src/com/paulasmuth/sqltap/CTreeIndex.scala @@ -36,11 +36,12 @@ object CTreeIndex { score += ctree.base_score Logger.debug("CTree: evaluating candidate: '" + ctree.name + - "' (score: " + score + ", cost: " + cost + ") for: " + root.resource_name) + "' (score: " + score + ", cost: " + cost + ") for: " + + root.resource_name) - var matches = (cost == 0 && winner_cost > 0) - matches ||= (score > top_score && winner_cost != 0) - matches ||= (score == top_score && cost > winner_cost) + val matches = (cost == 0 && winner_cost > 0) || + (score > top_score && winner_cost != 0) || + (score == top_score && cost > winner_cost) if (matches) { winner = ctree From 22869f1a6ae2310ee17761bccb81a8baaf8b3aa8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:39:37 +0200 Subject: [PATCH 32/75] generalize sending simple text HTTP responses --- src/com/paulasmuth/sqltap/HTTPConnection.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/com/paulasmuth/sqltap/HTTPConnection.scala index b146876..85ab50d 100644 --- a/src/com/paulasmuth/sqltap/HTTPConnection.scala +++ b/src/com/paulasmuth/sqltap/HTTPConnection.scala @@ -216,22 +216,28 @@ class HTTPConnection(sock: SocketChannel, worker: Worker) extends ReadyCallback[ Statistics.incr('http_requests_per_second) } - private def execute_ping() : Unit = { + private def execute_text(code: Integer, text: String) : Unit = { val http_buf = new HTTPWriter(buf) buf.clear - http_buf.write_status(200) - http_buf.write_content_length(6) + val body = text.getBytes + + http_buf.write_status(code) + http_buf.write_content_length(body.length) http_buf.write_default_headers() http_buf.finish_headers() - buf.put("pong\r\n".getBytes) + buf.put(body) buf.flip worker.requests_success.incrementAndGet() flush() } + private def execute_ping() : Unit = { + execute_text(200, "pong\r\n") + } + private def execute_request(params: List[String]) : Unit = { if (seq > 1) worker.requests_queued.incrementAndGet() From 57dedd08b3a85cfbbd5263ff099ec4935814954d Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:40:36 +0200 Subject: [PATCH 33/75] add the ability to optionally log Sqltap queries. CLI- and runtime configurable. --- src/com/paulasmuth/sqltap/Config.scala | 1 + src/com/paulasmuth/sqltap/HTTPConnection.scala | 10 ++++++++++ src/com/paulasmuth/sqltap/QueryParser.scala | 5 +++++ src/com/paulasmuth/sqltap/SQLTap.scala | 3 +++ 4 files changed, 19 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/com/paulasmuth/sqltap/Config.scala index 3cc2b13..bb5f093 100644 --- a/src/com/paulasmuth/sqltap/Config.scala +++ b/src/com/paulasmuth/sqltap/Config.scala @@ -27,6 +27,7 @@ object Config { 'memcache_host -> "127.0.0.1", 'memcache_port -> "11211", 'threads -> "4", + 'log_queries -> "false", 'expiration_handler -> "purge", 'cache_backend -> "memcache" ) diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/com/paulasmuth/sqltap/HTTPConnection.scala index 85ab50d..cdb282c 100644 --- a/src/com/paulasmuth/sqltap/HTTPConnection.scala +++ b/src/com/paulasmuth/sqltap/HTTPConnection.scala @@ -209,6 +209,16 @@ class HTTPConnection(sock: SocketChannel, worker: Worker) extends ReadyCallback[ else if (route.length > 1 && route.head == "expire") execute_expire(route.tail) + else if (route.length == 1 && route.head == "log_queries") { + Config.set('log_queries, "true") + execute_text(200, "sqltap query logging enabled\r\n") + } + + else if (route.length == 1 && route.head == "no_log_queries") { + Config.set('log_queries, "false") + execute_text(200, "sqltap query logging disabled\r\n") + } + else http_error(404, "not found") diff --git a/src/com/paulasmuth/sqltap/QueryParser.scala b/src/com/paulasmuth/sqltap/QueryParser.scala index 2fe6edc..b56459e 100644 --- a/src/com/paulasmuth/sqltap/QueryParser.scala +++ b/src/com/paulasmuth/sqltap/QueryParser.scala @@ -18,6 +18,11 @@ object QueryParser { private val PARSER_STATE_BODY = 5 def parse(stack: InstructionStack, qry: String) : Unit = { + + if (Config.get('log_queries).equals("true")) { + Logger.info("[QueryParser] parse: " + qry) + } + var args = new ListBuffer[String]() var state = PARSER_STATE_NEXT diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index e6152b2..b575d07 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -80,6 +80,9 @@ object SQLTap{ else if ((args(n) == "-t") || (args(n) == "--threads")) { Config.set('threads, args(n+1)); n += 2 } + else if (args(n) == "--log-queries") + { Config.set('log_queries, "true"); n += 1 } + else if ((args(n) == "-c") || (args(n) == "--config")) { Config.set('config_base, args(n+1)); n += 2 } From 12d6f47af0b0747b22079be4d0c61ae3286b32a8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:42:01 +0200 Subject: [PATCH 34/75] bump scala version to 2.11.7 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index bb30c99..d6783d9 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.11.4" +scalaVersion := "2.11.7" assemblySettings From bc7a0e427be3dfefdce6205d1d614ba8bc879b4e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 8 Oct 2015 23:31:11 +0200 Subject: [PATCH 35/75] adds SQL slow query statistics --- src/com/paulasmuth/sqltap/Statistics.scala | 3 +++ src/com/paulasmuth/sqltap/mysql/SQLQuery.scala | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Statistics.scala b/src/com/paulasmuth/sqltap/Statistics.scala index f943ef8..f795129 100644 --- a/src/com/paulasmuth/sqltap/Statistics.scala +++ b/src/com/paulasmuth/sqltap/Statistics.scala @@ -21,6 +21,9 @@ object Statistics { 'sql_requests_total -> new IntegralStatistic, 'sql_requests_per_second -> new DeltaStatistic, 'sql_request_time_mean -> new MeanStatistic, + 'sql_slow_queries_total -> new IntegralStatistic, + 'sql_slow_queries_per_second -> new DeltaStatistic, + 'sql_slow_queries_time_mean -> new MeanStatistic, 'memcache_requests_total -> new IntegralStatistic, 'memcache_requests_per_second -> new DeltaStatistic, 'memcache_connections_open -> new IntegralStatistic diff --git a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala b/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala index deeac0e..7c896b3 100644 --- a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala +++ b/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala @@ -49,6 +49,10 @@ class SQLQuery(query_str: String) extends TimeoutCallback { if (Config.has_key('log_slow_queries) && runtime_millis >= Config.get('log_slow_queries).toInt) { + Statistics.incr('sql_slow_queries_total) + Statistics.incr('sql_slow_queries_per_second) + Statistics.incr('sql_slow_queries_time_mean, runtime_millis) + Logger.log("[SQL] [Slow Query] (" + runtime_millis + "ms): " + query) } } From adc1fbbc249622889c38d820a44fd6ab00785232 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 23 Oct 2015 11:43:41 +0200 Subject: [PATCH 36/75] DeltaStatistic: code readability improvements --- src/com/paulasmuth/sqltap/DeltaStatistic.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/com/paulasmuth/sqltap/DeltaStatistic.scala b/src/com/paulasmuth/sqltap/DeltaStatistic.scala index 8a8b45b..c1b4da9 100644 --- a/src/com/paulasmuth/sqltap/DeltaStatistic.scala +++ b/src/com/paulasmuth/sqltap/DeltaStatistic.scala @@ -11,25 +11,23 @@ import java.util.concurrent.atomic.{AtomicInteger} import java.text.{DecimalFormat} class DeltaStatistic extends Statistic { - - private val bucket = new AtomicInteger() - private var value : Double = 0.0 + private val current = new AtomicInteger() + private var last : Double = 0.0 private val format = new DecimalFormat("0.00") def incr(delta: Double) : Unit= { - bucket.getAndAdd(delta.toInt) + current.getAndAdd(delta.toInt) } def decr(delta: Double) : Unit = { - bucket.getAndAdd(delta.toInt * -1) + current.getAndAdd(delta.toInt * -1) } def get() : String = { - format.format(value) + format.format(last) } def flush(f: Double) : Unit = { - value = bucket.getAndSet(0) / f + last = current.getAndSet(0) / f } - } From d66b3a204305ab5668c8d0695de7e51a3478df26 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 12:18:33 +0200 Subject: [PATCH 37/75] move src/* to convention src/main/scala/* --- build.sbt | 2 -- .../scala}/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTree.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeCache.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeIndex.scala | 0 .../scala}/com/paulasmuth/sqltap/CTreeInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeMarshal.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheAdapter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackend.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheBackendFactory.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheGetRequest.scala | 0 .../scala}/com/paulasmuth/sqltap/CachePurgeRequest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheRequest.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheStoreRequest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Config.scala | 0 .../scala}/com/paulasmuth/sqltap/CountInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/DeltaStatistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ElasticBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Exceptions.scala | 0 .../scala}/com/paulasmuth/sqltap/ExpirationHandler.scala | 0 .../scala}/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationJob.scala | 0 .../scala}/com/paulasmuth/sqltap/FindMultiInstruction.scala | 0 .../scala}/com/paulasmuth/sqltap/FindSingleInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/GZIPTranscoder.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPConnection.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPParser.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPWriter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Instruction.scala | 0 .../scala}/com/paulasmuth/sqltap/InstructionFactory.scala | 0 .../scala}/com/paulasmuth/sqltap/InstructionStack.scala | 0 .../scala}/com/paulasmuth/sqltap/IntegralStatistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/JSONWriter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Logger.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Manifest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/MeanStatistic.scala | 0 .../scala}/com/paulasmuth/sqltap/MemcacheConnection.scala | 0 .../scala}/com/paulasmuth/sqltap/MemcacheConnectionPool.scala | 0 .../scala}/com/paulasmuth/sqltap/NoopCacheBackend.scala | 0 .../scala}/com/paulasmuth/sqltap/NoopExpirationHandler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/PhiInstruction.scala | 0 .../scala}/com/paulasmuth/sqltap/PurgeExpirationHandler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Query.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/QueryParser.scala | 0 .../scala}/com/paulasmuth/sqltap/RawSQLInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ReadyCallback.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Record.scala | 0 .../scala}/com/paulasmuth/sqltap/RecordLookupJob.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/RelationTrace.scala | 0 .../scala}/com/paulasmuth/sqltap/ReplicationFeed.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Request.scala | 0 .../scala}/com/paulasmuth/sqltap/ResourceManifest.scala | 0 .../scala}/com/paulasmuth/sqltap/ResourceRelation.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLBuilder.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLHelper.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLTap.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Server.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Statistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Statistics.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Timeout.scala | 0 .../scala}/com/paulasmuth/sqltap/TimeoutCallback.scala | 0 .../scala}/com/paulasmuth/sqltap/TimeoutScheduler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Watchdog.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Worker.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/WrappedBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/XMLHelper.scala | 0 .../com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala | 0 .../com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinaryInteger.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinaryString.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala | 0 .../paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/HandshakePacket.scala | 0 .../com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala | 0 .../com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala | 0 .../com/paulasmuth/sqltap/mysql/LengthEncodedString.scala | 0 .../com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/PingPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLConnection.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLPacket.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLQuery.scala | 0 .../paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala | 0 .../com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala | 0 .../com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala | 0 92 files changed, 2 deletions(-) rename src/{ => main/scala}/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTree.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeCache.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeIndex.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeMarshal.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheAdapter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackend.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackendFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheGetRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CachePurgeRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheStoreRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Config.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CountInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/DeltaStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ElasticBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Exceptions.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationJob.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/FindMultiInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/FindSingleInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/GZIPTranscoder.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPParser.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPWriter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Instruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/InstructionFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/InstructionStack.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/IntegralStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/JSONWriter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Logger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Manifest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MeanStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MemcacheConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MemcacheConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/NoopCacheBackend.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/NoopExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/PhiInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/PurgeExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Query.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/QueryParser.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RawSQLInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ReadyCallback.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Record.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RecordLookupJob.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RelationTrace.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ReplicationFeed.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Request.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ResourceManifest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ResourceRelation.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLBuilder.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLHelper.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLTap.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Server.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Statistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Statistics.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Timeout.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/TimeoutCallback.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/TimeoutScheduler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Watchdog.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Worker.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/WrappedBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/XMLHelper.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinaryInteger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinaryString.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/HandshakePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/PingPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLQuery.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala (100%) diff --git a/build.sbt b/build.sbt index d6783d9..c48479e 100644 --- a/build.sbt +++ b/build.sbt @@ -8,8 +8,6 @@ version := "0.8.0" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") -scalaSource in Compile <<= baseDirectory(_ / "src") - scalaVersion := "2.11.7" assemblySettings diff --git a/src/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala diff --git a/src/com/paulasmuth/sqltap/CTree.scala b/src/main/scala/com/paulasmuth/sqltap/CTree.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTree.scala rename to src/main/scala/com/paulasmuth/sqltap/CTree.scala diff --git a/src/com/paulasmuth/sqltap/CTreeBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeBuffer.scala diff --git a/src/com/paulasmuth/sqltap/CTreeCache.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeCache.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala diff --git a/src/com/paulasmuth/sqltap/CTreeIndex.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeIndex.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeIndex.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeIndex.scala diff --git a/src/com/paulasmuth/sqltap/CTreeInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeInstruction.scala diff --git a/src/com/paulasmuth/sqltap/CTreeMarshal.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeMarshal.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeMarshal.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeMarshal.scala diff --git a/src/com/paulasmuth/sqltap/CacheAdapter.scala b/src/main/scala/com/paulasmuth/sqltap/CacheAdapter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheAdapter.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheAdapter.scala diff --git a/src/com/paulasmuth/sqltap/CacheBackend.scala b/src/main/scala/com/paulasmuth/sqltap/CacheBackend.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheBackend.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheBackend.scala diff --git a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala b/src/main/scala/com/paulasmuth/sqltap/CacheBackendFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheBackendFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheBackendFactory.scala diff --git a/src/com/paulasmuth/sqltap/CacheGetRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheGetRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheGetRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheGetRequest.scala diff --git a/src/com/paulasmuth/sqltap/CachePurgeRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CachePurgeRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CachePurgeRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CachePurgeRequest.scala diff --git a/src/com/paulasmuth/sqltap/CacheRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheRequest.scala diff --git a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheStoreRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheStoreRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheStoreRequest.scala diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/main/scala/com/paulasmuth/sqltap/Config.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Config.scala rename to src/main/scala/com/paulasmuth/sqltap/Config.scala diff --git a/src/com/paulasmuth/sqltap/CountInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/CountInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CountInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/CountInstruction.scala diff --git a/src/com/paulasmuth/sqltap/DeltaStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/DeltaStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/DeltaStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/DeltaStatistic.scala diff --git a/src/com/paulasmuth/sqltap/ElasticBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/ElasticBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ElasticBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/ElasticBuffer.scala diff --git a/src/com/paulasmuth/sqltap/Exceptions.scala b/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Exceptions.scala rename to src/main/scala/com/paulasmuth/sqltap/Exceptions.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationJob.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala diff --git a/src/com/paulasmuth/sqltap/FindMultiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/FindMultiInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala diff --git a/src/com/paulasmuth/sqltap/FindSingleInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/FindSingleInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala diff --git a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala b/src/main/scala/com/paulasmuth/sqltap/GZIPTranscoder.scala similarity index 100% rename from src/com/paulasmuth/sqltap/GZIPTranscoder.scala rename to src/main/scala/com/paulasmuth/sqltap/GZIPTranscoder.scala diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPConnection.scala diff --git a/src/com/paulasmuth/sqltap/HTTPParser.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPParser.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPParser.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPParser.scala diff --git a/src/com/paulasmuth/sqltap/HTTPWriter.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPWriter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPWriter.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPWriter.scala diff --git a/src/com/paulasmuth/sqltap/Instruction.scala b/src/main/scala/com/paulasmuth/sqltap/Instruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Instruction.scala rename to src/main/scala/com/paulasmuth/sqltap/Instruction.scala diff --git a/src/com/paulasmuth/sqltap/InstructionFactory.scala b/src/main/scala/com/paulasmuth/sqltap/InstructionFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/InstructionFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/InstructionFactory.scala diff --git a/src/com/paulasmuth/sqltap/InstructionStack.scala b/src/main/scala/com/paulasmuth/sqltap/InstructionStack.scala similarity index 100% rename from src/com/paulasmuth/sqltap/InstructionStack.scala rename to src/main/scala/com/paulasmuth/sqltap/InstructionStack.scala diff --git a/src/com/paulasmuth/sqltap/IntegralStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/IntegralStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/IntegralStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/IntegralStatistic.scala diff --git a/src/com/paulasmuth/sqltap/JSONWriter.scala b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/JSONWriter.scala rename to src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala diff --git a/src/com/paulasmuth/sqltap/Logger.scala b/src/main/scala/com/paulasmuth/sqltap/Logger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Logger.scala rename to src/main/scala/com/paulasmuth/sqltap/Logger.scala diff --git a/src/com/paulasmuth/sqltap/Manifest.scala b/src/main/scala/com/paulasmuth/sqltap/Manifest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Manifest.scala rename to src/main/scala/com/paulasmuth/sqltap/Manifest.scala diff --git a/src/com/paulasmuth/sqltap/MeanStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/MeanStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MeanStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/MeanStatistic.scala diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/main/scala/com/paulasmuth/sqltap/MemcacheConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MemcacheConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/MemcacheConnection.scala diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/MemcacheConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/MemcacheConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/NoopCacheBackend.scala b/src/main/scala/com/paulasmuth/sqltap/NoopCacheBackend.scala similarity index 100% rename from src/com/paulasmuth/sqltap/NoopCacheBackend.scala rename to src/main/scala/com/paulasmuth/sqltap/NoopCacheBackend.scala diff --git a/src/com/paulasmuth/sqltap/NoopExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/NoopExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/NoopExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/NoopExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/PhiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/PhiInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/PhiInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/PhiInstruction.scala diff --git a/src/com/paulasmuth/sqltap/PurgeExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/PurgeExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/PurgeExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/PurgeExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/Query.scala b/src/main/scala/com/paulasmuth/sqltap/Query.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Query.scala rename to src/main/scala/com/paulasmuth/sqltap/Query.scala diff --git a/src/com/paulasmuth/sqltap/QueryParser.scala b/src/main/scala/com/paulasmuth/sqltap/QueryParser.scala similarity index 100% rename from src/com/paulasmuth/sqltap/QueryParser.scala rename to src/main/scala/com/paulasmuth/sqltap/QueryParser.scala diff --git a/src/com/paulasmuth/sqltap/RawSQLInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/RawSQLInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RawSQLInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/RawSQLInstruction.scala diff --git a/src/com/paulasmuth/sqltap/ReadyCallback.scala b/src/main/scala/com/paulasmuth/sqltap/ReadyCallback.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ReadyCallback.scala rename to src/main/scala/com/paulasmuth/sqltap/ReadyCallback.scala diff --git a/src/com/paulasmuth/sqltap/Record.scala b/src/main/scala/com/paulasmuth/sqltap/Record.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Record.scala rename to src/main/scala/com/paulasmuth/sqltap/Record.scala diff --git a/src/com/paulasmuth/sqltap/RecordLookupJob.scala b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RecordLookupJob.scala rename to src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala diff --git a/src/com/paulasmuth/sqltap/RelationTrace.scala b/src/main/scala/com/paulasmuth/sqltap/RelationTrace.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RelationTrace.scala rename to src/main/scala/com/paulasmuth/sqltap/RelationTrace.scala diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ReplicationFeed.scala rename to src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala diff --git a/src/com/paulasmuth/sqltap/Request.scala b/src/main/scala/com/paulasmuth/sqltap/Request.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Request.scala rename to src/main/scala/com/paulasmuth/sqltap/Request.scala diff --git a/src/com/paulasmuth/sqltap/ResourceManifest.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ResourceManifest.scala rename to src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala diff --git a/src/com/paulasmuth/sqltap/ResourceRelation.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceRelation.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ResourceRelation.scala rename to src/main/scala/com/paulasmuth/sqltap/ResourceRelation.scala diff --git a/src/com/paulasmuth/sqltap/SQLBuilder.scala b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLBuilder.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala diff --git a/src/com/paulasmuth/sqltap/SQLHelper.scala b/src/main/scala/com/paulasmuth/sqltap/SQLHelper.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLHelper.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLHelper.scala diff --git a/src/com/paulasmuth/sqltap/SQLInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/SQLInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLInstruction.scala diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLTap.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLTap.scala diff --git a/src/com/paulasmuth/sqltap/Server.scala b/src/main/scala/com/paulasmuth/sqltap/Server.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Server.scala rename to src/main/scala/com/paulasmuth/sqltap/Server.scala diff --git a/src/com/paulasmuth/sqltap/Statistic.scala b/src/main/scala/com/paulasmuth/sqltap/Statistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Statistic.scala rename to src/main/scala/com/paulasmuth/sqltap/Statistic.scala diff --git a/src/com/paulasmuth/sqltap/Statistics.scala b/src/main/scala/com/paulasmuth/sqltap/Statistics.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Statistics.scala rename to src/main/scala/com/paulasmuth/sqltap/Statistics.scala diff --git a/src/com/paulasmuth/sqltap/Timeout.scala b/src/main/scala/com/paulasmuth/sqltap/Timeout.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Timeout.scala rename to src/main/scala/com/paulasmuth/sqltap/Timeout.scala diff --git a/src/com/paulasmuth/sqltap/TimeoutCallback.scala b/src/main/scala/com/paulasmuth/sqltap/TimeoutCallback.scala similarity index 100% rename from src/com/paulasmuth/sqltap/TimeoutCallback.scala rename to src/main/scala/com/paulasmuth/sqltap/TimeoutCallback.scala diff --git a/src/com/paulasmuth/sqltap/TimeoutScheduler.scala b/src/main/scala/com/paulasmuth/sqltap/TimeoutScheduler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/TimeoutScheduler.scala rename to src/main/scala/com/paulasmuth/sqltap/TimeoutScheduler.scala diff --git a/src/com/paulasmuth/sqltap/Watchdog.scala b/src/main/scala/com/paulasmuth/sqltap/Watchdog.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Watchdog.scala rename to src/main/scala/com/paulasmuth/sqltap/Watchdog.scala diff --git a/src/com/paulasmuth/sqltap/Worker.scala b/src/main/scala/com/paulasmuth/sqltap/Worker.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Worker.scala rename to src/main/scala/com/paulasmuth/sqltap/Worker.scala diff --git a/src/com/paulasmuth/sqltap/WrappedBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/WrappedBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/WrappedBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/WrappedBuffer.scala diff --git a/src/com/paulasmuth/sqltap/XMLHelper.scala b/src/main/scala/com/paulasmuth/sqltap/XMLHelper.scala similarity index 100% rename from src/com/paulasmuth/sqltap/XMLHelper.scala rename to src/main/scala/com/paulasmuth/sqltap/XMLHelper.scala diff --git a/src/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinaryInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinaryInteger.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinaryString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryString.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinaryString.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinaryString.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala diff --git a/src/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/HandshakePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/HandshakePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/HandshakePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/HandshakePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala diff --git a/src/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala diff --git a/src/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala diff --git a/src/com/paulasmuth/sqltap/mysql/PingPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/PingPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/PingPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/PingPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLConnection.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLQuery.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLQuery.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLQuery.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala diff --git a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala From 657cd1524fd18333b10336d3af9c26567ea09b4d Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:16:23 +0200 Subject: [PATCH 38/75] add scalatest as test dependency --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index c48479e..aa2e091 100644 --- a/build.sbt +++ b/build.sbt @@ -17,3 +17,4 @@ jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" +libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test" From 860f2d645e634e332594b6d4cfe9f9552a21b21e Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:18:41 +0200 Subject: [PATCH 39/75] add specs for existing JSONWriter behaviour --- .../paulasmuth/sqltap/JSONWriterSpec.scala | 43 +++++++++++++++++++ .../com/paulasmuth/sqltap/UnitSpec.scala | 7 +++ 2 files changed, 50 insertions(+) create mode 100644 src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala create mode 100644 src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala diff --git a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala new file mode 100644 index 0000000..3201c7d --- /dev/null +++ b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala @@ -0,0 +1,43 @@ +package com.paulasmuth.sqltap + +import java.nio.ByteBuffer + +class JSONWriterSpec extends UnitSpec { + + describe("write_escaped") { + it("0x20 -> ' '") { + val str = write_escaped(Array(0x20.toByte)) + assert(str == " ") + } + + it("0xA -> '\\n'") { + val str = write_escaped(Array(0xA.toByte)) + assert(str == "\\n") + } + + it("0x22 -> '\\\"'") { + val str = write_escaped(Array(0x22.toByte)) + assert(str == "\\\"") + } + + it("0x5C -> '\\\\'") { + val str = write_escaped(Array(0x5C.toByte)) + assert(str == "\\\\") + } + } + + def buffer_to_string(buf: ByteBuffer) : String = { + val bytes = new Array[Byte](buf.position()) + buf.rewind() + buf.get(bytes) + new String(bytes, "UTF-8") + } + + def write_escaped(in_bytes: Array[Byte]) : String = { + val buf = ByteBuffer.allocate(10) + val writer = new JSONWriter(new WrappedBuffer(buf)) + writer.write_escaped(new String(in_bytes, "UTF-8")) + buffer_to_string(buf) + } + +} diff --git a/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala b/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala new file mode 100644 index 0000000..e2bc883 --- /dev/null +++ b/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala @@ -0,0 +1,7 @@ +package com.paulasmuth.sqltap + +import org.scalatest.FunSpec +import org.scalatest.BeforeAndAfter +import org.scalatest.Assertions._ + +abstract class UnitSpec extends FunSpec with BeforeAndAfter From 841b35250712d42ca10a44a0102a00d9ade71863 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:44:29 +0200 Subject: [PATCH 40/75] add json string escaping for bytes < 0x20 --- src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala | 5 ++++- src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala index 2902650..576ce3b 100644 --- a/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala +++ b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala @@ -86,7 +86,10 @@ class JSONWriter(buf: WrappedBuffer) { buf.write(Array(0x5C.toByte, 0x22.toByte)) // \" } else if (b == 0x5C) { buf.write(Array(0x5C.toByte, 0x5C.toByte)) // \\ - } else if ((b == 0) || ((b >= 0x20))) { + } else if (b < 0x20) { + buf.write(Array(0x5C.toByte, 0x75.toByte)) + buf.write("%04x".format(b).getBytes) // \u000b + } else { buf.write(byte) } } diff --git a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala index 3201c7d..0e2866b 100644 --- a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala +++ b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala @@ -24,6 +24,11 @@ class JSONWriterSpec extends UnitSpec { val str = write_escaped(Array(0x5C.toByte)) assert(str == "\\\\") } + + it("0xB -> '\\u000b'") { + val str = write_escaped(Array(0xB.toByte)) + assert(str == "\\u000b") + } } def buffer_to_string(buf: ByteBuffer) : String = { From 3522f1fa9fedb7939559a1560a5f87a62669a2d0 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 2 Nov 2015 14:59:55 +0100 Subject: [PATCH 41/75] add escaping for default_order field --- src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala index 6471c98..af2df00 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala @@ -31,7 +31,7 @@ class ResourceManifest(doc: xml.Node) { elem.attr("id_field", false, "id") val default_order : String = - elem.attr("default_order", false, id_field + " DESC") + elem.attr("default_order", false, "`" + id_field + "` DESC") val relations = ((List[ResourceRelation]() /: (doc \ "relation")) From cebe5cd5f1402a1df00e1a7ba9d8242a7aad4396 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Nov 2015 12:22:25 +0100 Subject: [PATCH 42/75] build.sbt: compile fix --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index aa2e091..8b8d79d 100644 --- a/build.sbt +++ b/build.sbt @@ -17,4 +17,5 @@ jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" + libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test" From 8cd2bdd6633bcbd6c308b3872434afe7662a2340 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Nov 2015 12:22:36 +0100 Subject: [PATCH 43/75] version bump to 0.8.1 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8b8d79d..6041778 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.0" +version := "0.8.1" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") From 5cac6f5630ce037d3c496ece4449ace47f2dcc9b Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 22 Jan 2016 15:47:18 +0100 Subject: [PATCH 44/75] Dockerize --- .dockerignore | 3 +++ Dockerfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++ sqltap.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 sqltap.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..35706e6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.gitignore +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e0cee51 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +FROM java:8 + +MAINTAINER DaWanda + +ENV SCALA_VERSION="2.11.7" \ + SBT_VERSION="0.13.8" \ + SQLTAP_VERSION="0.8.1" + +ADD http://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz /tmp/scala.tgz +RUN tar -C /opt -xzf /tmp/scala.tgz && rm -f /tmp/scala.tgz + +ENV PATH="/opt/scala-$SCALA_VERSION/bin:$PATH" + +ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb +RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb +RUN apt-get update && apt-get install sbt + +RUN sbt + +ADD . /opt/sqltap +WORKDIR /opt/sqltap +RUN sbt assembly + +ENV SQLTAP_HTTP_PORT="3000" \ + SQLTAP_THREADS="16" \ + SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ + SQLTAP_SCHEMA="/var/schema.xml" \ + SQLTAP_OPTS="" \ + MYSQL_HOST="127.0.0.1" \ + MYSQL_PORT="3306" \ + MYSQL_USER="fetch" \ + MYSQL_DATABASE="test" \ + MYSQL_NUMCONNS="6" \ + MYSQL_QUEUELEN="2500" \ + JMX_PORT="9191" \ + RMI_BIND="127.0.0.1" \ + JAVA_XMX="16384M" \ + HOSTNAME="sqltap1" \ + CACHE_BACKEND="memcache" \ + MEMCACHE_HOST="127.0.0.1" \ + MEMCACHE_PORT="11211" \ + MEMCACHE_QUEUELEN="8192" \ + MEMCACHE_NUMCONNS="20" + +EXPOSE 3000 + +CMD ["/opt/sqltap/sqltap.sh"] diff --git a/sqltap.sh b/sqltap.sh new file mode 100755 index 0000000..fdde1c8 --- /dev/null +++ b/sqltap.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +if [[ "$SCHEMA_URL" != "" ]]; then + curl -sL "$SCHEMA_URL" -o /var/schema.xml +fi + +opts="" +opts="$opts --config '${SQLTAP_SCHEMA}'" +opts="$opts --http ${SQLTAP_HTTP_PORT}" +opts="$opts --disable-keepalive" +opts="$opts -t ${SQLTAP_THREADS}" +opts="$opts --mysql-host ${MYSQL_HOST}" +opts="$opts --mysql-port ${MYSQL_PORT}" +opts="$opts --mysql-user ${MYSQL_USER}" +opts="$opts --mysql-database ${MYSQL_DATABASE}" +opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}" +opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}" +opts="$opts --cache-backend ${CACHE_BACKEND}" +if [[ "${CACHE_BACKEND}" == "memcache" ]]; then + opts="$opts --memcache-host ${MEMCACHE_HOST}" + opts="$opts --memcache-port ${MEMCACHE_PORT}" + opts="$opts --memcache-queuelen ${MEMCACHE_QUEUELEN}" + opts="$opts --memcache-numconns ${MEMCACHE_NUMCONNS}" +fi +opts="$opts ${SQLTAP_OPTS}" + +exec java \ + -Djava.rmi.server.hostname="${RMI_BIND}" \ + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port="${JMX_PORT}" \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Xmx"${JAVA_XMX}" -XX:GCTimeRatio=99 -XX:+UseConcMarkSweepGC \ + -jar "${SQLTAP_JARFILE}" \ + $opts From 22c6318d46397ef739ec4efff0adf91a0d24b825 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 22 Jan 2016 17:31:43 +0100 Subject: [PATCH 45/75] Dockerfile: adds missing ENV --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e0cee51..8764ea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ SQLTAP_SCHEMA="/var/schema.xml" \ + SCHEMA_URL="" \ SQLTAP_OPTS="" \ MYSQL_HOST="127.0.0.1" \ MYSQL_PORT="3306" \ From cf76c52ee22d477e735580a7eda4981af3dc93b9 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 02:13:58 +0100 Subject: [PATCH 46/75] Dockerfile refactoring, killing unnecessary statements and do some cleanup --- Dockerfile | 36 ++++++++++++++---------------------- build.sbt | 2 +- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8764ea0..bf29d3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,12 @@ FROM java:8 - MAINTAINER DaWanda -ENV SCALA_VERSION="2.11.7" \ - SBT_VERSION="0.13.8" \ - SQLTAP_VERSION="0.8.1" - -ADD http://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz /tmp/scala.tgz -RUN tar -C /opt -xzf /tmp/scala.tgz && rm -f /tmp/scala.tgz - -ENV PATH="/opt/scala-$SCALA_VERSION/bin:$PATH" - -ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb -RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb -RUN apt-get update && apt-get install sbt - -RUN sbt - -ADD . /opt/sqltap -WORKDIR /opt/sqltap -RUN sbt assembly +ARG SBT_VERSION="0.13.8" +ARG SQLTAP_JARFILE="/usr/lib/sqltap.jar" ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ - SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ - SQLTAP_SCHEMA="/var/schema.xml" \ + SQLTAP_SCHEMA="/etc/sqltap-schema.xml" \ SCHEMA_URL="" \ SQLTAP_OPTS="" \ MYSQL_HOST="127.0.0.1" \ @@ -43,6 +25,16 @@ ENV SQLTAP_HTTP_PORT="3000" \ MEMCACHE_QUEUELEN="8192" \ MEMCACHE_NUMCONNS="20" -EXPOSE 3000 +ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb +RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb +RUN apt-get update && apt-get install sbt + +ADD . /opt/sqltap +RUN cd /opt/sqltap && \ + sbt assembly && \ + cp -vpi /opt/sqltap/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + rm -rf /opt/sqltap + +EXPOSE $SQLTAP_HTTP_PORT CMD ["/opt/sqltap/sqltap.sh"] diff --git a/build.sbt b/build.sbt index 6041778..9ce4fdd 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ scalaVersion := "2.11.7" assemblySettings -jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } +jarName in assembly := { s"${name.value.toLowerCase}.jar" } fork in run := true From f35d059964dc0474ad9c891eb05e070513bcccb3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:09:49 +0100 Subject: [PATCH 47/75] docker: improved bootup script to be a little more user-friendly --- sqltap.sh => bootup.sh | 47 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) rename sqltap.sh => bootup.sh (55%) diff --git a/sqltap.sh b/bootup.sh similarity index 55% rename from sqltap.sh rename to bootup.sh index fdde1c8..ace9dfc 100755 --- a/sqltap.sh +++ b/bootup.sh @@ -1,9 +1,47 @@ #!/bin/bash +set -e if [[ "$SCHEMA_URL" != "" ]]; then - curl -sL "$SCHEMA_URL" -o /var/schema.xml + curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" fi +require_arg() { + local name="$1" + local val=`eval "echo \\\$${name}"` + + echo "${name}: ${val}" + + if [[ "${val}" == "" ]]; then + echo "Error. Required argument ${name} missing." 1>&2 + exit 1 + fi +} + +require_args() { + while [[ $# -ne 0 ]]; do + require_arg $1 + shift + done +} + +require_args SQLTAP_SCHEMA \ + MYSQL_PORT \ + SQLTAP_HTTP_PORT \ + SQLTAP_THREADS \ + SQLTAP_SCHEMA \ + SCHEMA_URL \ + SQLTAP_OPTS \ + MYSQL_HOST \ + MYSQL_PORT \ + MYSQL_USER \ + MYSQL_DATABASE \ + MYSQL_NUMCONNS \ + MYSQL_QUEUELEN \ + JMX_PORT \ + RMI_BIND \ + JAVA_XMX \ + CACHE_BACKEND + opts="" opts="$opts --config '${SQLTAP_SCHEMA}'" opts="$opts --http ${SQLTAP_HTTP_PORT}" @@ -16,12 +54,19 @@ opts="$opts --mysql-database ${MYSQL_DATABASE}" opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}" opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}" opts="$opts --cache-backend ${CACHE_BACKEND}" + if [[ "${CACHE_BACKEND}" == "memcache" ]]; then + require_args MEMCACHE_HOST \ + MEMCACHE_PORT \ + MEMCACHE_QUEUELEN \ + MEMCACHE_NUMCONNS + opts="$opts --memcache-host ${MEMCACHE_HOST}" opts="$opts --memcache-port ${MEMCACHE_PORT}" opts="$opts --memcache-queuelen ${MEMCACHE_QUEUELEN}" opts="$opts --memcache-numconns ${MEMCACHE_NUMCONNS}" fi + opts="$opts ${SQLTAP_OPTS}" exec java \ From c55d041f45ad2e026ab7173f4446c8d10c7ca54f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:10:40 +0100 Subject: [PATCH 48/75] .dockerignore update --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 35706e6..44c9325 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ .git .gitignore .DS_Store +/target +/bin From 24bcb8ffc932feef0585277264e1e67d94da51ea Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:33:05 +0100 Subject: [PATCH 49/75] Dockerfile: drop unnecessary HOSTNAME env var --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf29d3e..4836d00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ ENV SQLTAP_HTTP_PORT="3000" \ JMX_PORT="9191" \ RMI_BIND="127.0.0.1" \ JAVA_XMX="16384M" \ - HOSTNAME="sqltap1" \ CACHE_BACKEND="memcache" \ MEMCACHE_HOST="127.0.0.1" \ MEMCACHE_PORT="11211" \ From 7d833de61c5fe6bbd8bd521b1ded3975e9bf4bea Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:33:47 +0100 Subject: [PATCH 50/75] Dockerfile: remove default arg for MEMCACHE_HOST as it must be set explicitely when cache-backend is memcache --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4836d00..48409d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV SQLTAP_HTTP_PORT="3000" \ RMI_BIND="127.0.0.1" \ JAVA_XMX="16384M" \ CACHE_BACKEND="memcache" \ - MEMCACHE_HOST="127.0.0.1" \ + MEMCACHE_HOST="" \ MEMCACHE_PORT="11211" \ MEMCACHE_QUEUELEN="8192" \ MEMCACHE_NUMCONNS="20" From 14d6523ffc4aaa989766a54a177d240ea1e6875d Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:34:22 +0100 Subject: [PATCH 51/75] Dockerfile: use new startup location --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 48409d0..1492ff8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ RUN cd /opt/sqltap && \ EXPOSE $SQLTAP_HTTP_PORT -CMD ["/opt/sqltap/sqltap.sh"] +CMD ["/bootup.sh"] From 3d3787dca24136a6f71c80db8f10cbcb7e7e0d16 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:34:37 +0100 Subject: [PATCH 52/75] Dockerfile: improved build process --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1492ff8..4cb43ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,11 +28,15 @@ ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb RUN apt-get update && apt-get install sbt -ADD . /opt/sqltap -RUN cd /opt/sqltap && \ +ADD project /usr/src/project/ +ADD src /usr/src/src/ +ADD build.sbt /usr/src/ +ADD bootup.sh /bootup.sh + +RUN cd /usr/src && \ sbt assembly && \ - cp -vpi /opt/sqltap/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ - rm -rf /opt/sqltap + cp -vpi /usr/src/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + rm -rf /usr/src/* EXPOSE $SQLTAP_HTTP_PORT From eda87f942bc7b6f499c3801263efd31616f009f6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:36:57 +0100 Subject: [PATCH 53/75] adds Makefile for convinience --- Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..51e40f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# This file is part of the "sqltap" project, http://github.com/dawanda/sqltap> +# (c) 2016 Christian Parpart +# +# Licensed under the MIT License (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of +# the License at: http://opensource.org/licenses/MIT + +IMAGE = sqltap +VERSION = $(shell grep ^version build.sbt | cut -d\" -f2) + +image: + docker build -t ${IMAGE}:${VERSION} . + +push: image + docker push ${IMAGE}:${VERSION} + +clean: + rm -rf target + +.PHONY: image clean + +# vim:ts=8:noet From 8089a8a1a6cd3d83e15107ff6815dd0baa30fdcc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:18:21 +0100 Subject: [PATCH 54/75] bootup.sh: drop requirement of SQLTAP_OPTS env var --- bootup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bootup.sh b/bootup.sh index ace9dfc..b5e70c3 100755 --- a/bootup.sh +++ b/bootup.sh @@ -30,7 +30,6 @@ require_args SQLTAP_SCHEMA \ SQLTAP_THREADS \ SQLTAP_SCHEMA \ SCHEMA_URL \ - SQLTAP_OPTS \ MYSQL_HOST \ MYSQL_PORT \ MYSQL_USER \ From ddbf11904abd80c5c8499a7ab743a6de5f245c6b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:47:41 +0100 Subject: [PATCH 55/75] docker fixes --- Dockerfile | 2 +- bootup.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4cb43ad..8a839b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,4 +40,4 @@ RUN cd /usr/src && \ EXPOSE $SQLTAP_HTTP_PORT -CMD ["/bootup.sh"] +CMD ["/bootup.sh", "$SQLTAP_JARFILE"] diff --git a/bootup.sh b/bootup.sh index b5e70c3..059f2fa 100755 --- a/bootup.sh +++ b/bootup.sh @@ -1,5 +1,7 @@ #!/bin/bash -set -e +set -ex + +SQLTAP_JARFILE="${1}" if [[ "$SCHEMA_URL" != "" ]]; then curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" @@ -28,7 +30,6 @@ require_args SQLTAP_SCHEMA \ MYSQL_PORT \ SQLTAP_HTTP_PORT \ SQLTAP_THREADS \ - SQLTAP_SCHEMA \ SCHEMA_URL \ MYSQL_HOST \ MYSQL_PORT \ @@ -42,7 +43,7 @@ require_args SQLTAP_SCHEMA \ CACHE_BACKEND opts="" -opts="$opts --config '${SQLTAP_SCHEMA}'" +opts="$opts --config ${SQLTAP_SCHEMA}" opts="$opts --http ${SQLTAP_HTTP_PORT}" opts="$opts --disable-keepalive" opts="$opts -t ${SQLTAP_THREADS}" From ba78cc0ca6fe3eac9e00407b6138bed46414be03 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:55:18 +0100 Subject: [PATCH 56/75] docker: fixing SQLTAP_JARFILE --- Dockerfile | 5 ++--- bootup.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a839b7..b2fc0e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ FROM java:8 MAINTAINER DaWanda ARG SBT_VERSION="0.13.8" -ARG SQLTAP_JARFILE="/usr/lib/sqltap.jar" ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ @@ -35,9 +34,9 @@ ADD bootup.sh /bootup.sh RUN cd /usr/src && \ sbt assembly && \ - cp -vpi /usr/src/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + cp -vpi /usr/src/target/scala-*/sqltap.jar /usr/lib/sqltap.jar && \ rm -rf /usr/src/* EXPOSE $SQLTAP_HTTP_PORT -CMD ["/bootup.sh", "$SQLTAP_JARFILE"] +CMD ["/bootup.sh"] diff --git a/bootup.sh b/bootup.sh index 059f2fa..c77ae1c 100755 --- a/bootup.sh +++ b/bootup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -SQLTAP_JARFILE="${1}" +SQLTAP_JARFILE="/usr/lib/sqltap.jar" if [[ "$SCHEMA_URL" != "" ]]; then curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" From 040c0b87c454530c7bdb6381831bf457d7d36e08 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:56:12 +0100 Subject: [PATCH 57/75] docker: move ADD bootup.sh at the most bottom place as possible --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b2fc0e7..8a8d65e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ RUN apt-get update && apt-get install sbt ADD project /usr/src/project/ ADD src /usr/src/src/ ADD build.sbt /usr/src/ -ADD bootup.sh /bootup.sh RUN cd /usr/src && \ sbt assembly && \ @@ -39,4 +38,5 @@ RUN cd /usr/src && \ EXPOSE $SQLTAP_HTTP_PORT +ADD bootup.sh /bootup.sh CMD ["/bootup.sh"] From 344415bd4101dc2af933c187d7cfb9ffb0401dcf Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 25 Jan 2016 11:14:50 +0100 Subject: [PATCH 58/75] Makefile: adds a p[h]ony --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51e40f7..9c7b446 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,6 @@ push: image clean: rm -rf target -.PHONY: image clean +.PHONY: image clean push # vim:ts=8:noet From 393243bca72f8575491c0094e4d257fdbdec8d39 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 25 Jan 2016 11:15:15 +0100 Subject: [PATCH 59/75] bootup.sh: env var cleanup --- bootup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootup.sh b/bootup.sh index c77ae1c..9a04783 100755 --- a/bootup.sh +++ b/bootup.sh @@ -9,7 +9,7 @@ fi require_arg() { local name="$1" - local val=`eval "echo \\\$${name}"` + local val="${!name}" echo "${name}: ${val}" From 63ef610b5baf8e2693e8922a2e259a991b038e65 Mon Sep 17 00:00:00 2001 From: Paul Asmuth Date: Mon, 25 Jan 2016 17:03:04 +0100 Subject: [PATCH 60/75] untested statsd reporting example --- bootup.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bootup.sh b/bootup.sh index 9a04783..60a88a3 100755 --- a/bootup.sh +++ b/bootup.sh @@ -69,6 +69,17 @@ fi opts="$opts ${SQLTAP_OPTS}" +report_to_statsd() { + while sleep 1; do + curl "http://localhost:${SQLTAP_HTTP_PORT}/stats" | \ + sed -e 's/,/\n/g' | \ + sed -e 's/^[^"]*"//g' -e 's/": *"/:/g' -e 's/".*$//g' -e 's/^/myprefix./g' | \ + nc -u -w0 ${STATSD_HOST} ${STATSD_PORT} + done +} + +report_to_statsd& + exec java \ -Djava.rmi.server.hostname="${RMI_BIND}" \ -Dcom.sun.management.jmxremote \ From 1975517e5ecdb8fd2b98abe0ee671154d9af1514 Mon Sep 17 00:00:00 2001 From: Paul Asmuth Date: Wed, 2 Mar 2016 17:21:07 +0100 Subject: [PATCH 61/75] handle >31bit integer field ids --- src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala | 2 +- src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- .../com/paulasmuth/sqltap/FindMultiInstruction.scala | 6 +++--- .../com/paulasmuth/sqltap/FindSingleInstruction.scala | 4 ++-- src/main/scala/com/paulasmuth/sqltap/Record.scala | 8 ++++---- .../scala/com/paulasmuth/sqltap/RecordLookupJob.scala | 2 +- .../scala/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala index 234f58a..bf98e3d 100644 --- a/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala +++ b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala @@ -46,7 +46,7 @@ object CTreeCache { * @param record_id the primary id of the resource/record to be expired * @param resource_name the name of the resource to be expired */ - def expire(worker: Worker, resource_name: String, record_id: Int) : Unit = { + def expire(worker: Worker, resource_name: String, record_id: Long) : Unit = { if (!Manifest.has_resource(resource_name)) throw new ParseException("unknown resource: " + resource_name) diff --git a/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala index 354ec50..58716c2 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala @@ -34,7 +34,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] * * @param record the primary record id */ - def execute(record_id: Int) : Unit = { + def execute(record_id: Long) : Unit = { val primary_id = ctree.resource.id_field Logger.debug( diff --git a/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala index 9d7a62c..f597796 100644 --- a/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala +++ b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala @@ -29,7 +29,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { var offset : String = null var expanded : Boolean = false - var join_id : Int = 0 + var join_id : Long = 0 var join_conditions : String = null def execute(_worker: Worker) : Unit = { @@ -75,7 +75,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { else if (relation.join_field_local != null && prev.is_finished) { state = INS_STATE_READY - join_id = prev.record.get(relation.join_field_local).toInt + join_id = prev.record.get(relation.join_field_local).toLong } } @@ -186,7 +186,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { var found = false val this_id = row( - query.columns.indexOf(relation.resource.id_field)).toInt + query.columns.indexOf(relation.resource.id_field)).toLong while (!found && n > 0) { val this_ins = next(n - 1) diff --git a/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala index e53ca26..a5765f5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala +++ b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala @@ -26,7 +26,7 @@ class FindSingleInstruction extends SQLInstruction with CTreeInstruction { var order : String = null var join_field : String = null - var join_id : Int = 0 + var join_id : Long = 0 var allow_empty : Boolean = false def execute(_worker: Worker) : Unit = { @@ -66,7 +66,7 @@ class FindSingleInstruction extends SQLInstruction with CTreeInstruction { } state = INS_STATE_READY - join_id = join_id_str.toInt + join_id = join_id_str.toLong if (join_id == 0) { state = INS_STATE_DONE diff --git a/src/main/scala/com/paulasmuth/sqltap/Record.scala b/src/main/scala/com/paulasmuth/sqltap/Record.scala index b655062..6f094c4 100644 --- a/src/main/scala/com/paulasmuth/sqltap/Record.scala +++ b/src/main/scala/com/paulasmuth/sqltap/Record.scala @@ -15,16 +15,16 @@ class Record(_resource: ResourceManifest) { var fields = ListBuffer[String]() var data = ListBuffer[String]() - def id() : Int = { - get(resource.id_field).toInt + def id() : Long = { + get(resource.id_field).toLong } - def set_id(id: Int) : Unit = { + def set_id(id: Long) : Unit = { set(resource.id_field, id.toString) } def set_id(id: String) : Unit = { - set_id(id.toInt) + set_id(id.toLong) } def has_id() : Boolean = { diff --git a/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala index ca4030a..900cfa1 100644 --- a/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala +++ b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala @@ -14,7 +14,7 @@ class RecordLookupJob(worker: Worker, resource: ResourceManifest) extends ReadyC private val callbacks = new ListBuffer[ReadyCallback[Record]]() - def execute(record_id: Int) : Unit = { + def execute(record_id: Long) : Unit = { if (callbacks.length == 0) { return // RecordLookupJob is a noop without callbacks } diff --git a/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala index 52c988d..0ac2ac5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -29,7 +29,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { CTreeCache.expire(this, Manifest.resource_name_for_table(evt.table_name), - evt.primary_key.toInt) + evt.primary_key.toLong) } } diff --git a/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala index c074a3f..7ff6da7 100644 --- a/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala +++ b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala @@ -12,7 +12,7 @@ object SQLBuilder { def select( res: ResourceManifest, id_field: String, - id: Int, + id: Long, fields: List[String], cond: String, order: String, @@ -45,7 +45,7 @@ object SQLBuilder { def count( res: ResourceManifest, id_field: String, - id: Int, + id: Long, cond: String ) : String = ( From 7f0d6551577f0865c5870c321657bc512f3462f7 Mon Sep 17 00:00:00 2001 From: Paul Asmuth Date: Wed, 2 Mar 2016 17:38:04 +0100 Subject: [PATCH 62/75] also support 64bit integers in RowsBinlogEvent --- .../com/paulasmuth/sqltap/mysql/BinaryInteger.scala | 9 +++++++++ .../com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala index c3a9108..d67c571 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala @@ -18,4 +18,13 @@ object BinaryInteger { return value } + def readLong(data: Array[Byte], pos: Int, len: Int) : Long = { + var value : Long = 0 + + for (n <- (0 until len)) + value += (data(pos + n) & 0x000000ff) << (8*n) + + return value + } + } diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala index f4a954b..e9e6cc5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala @@ -65,8 +65,8 @@ trait RowsBinlogEvent extends BinlogEvent { num._1 } - def read_int(bytes: Int) : Int = { - val num = BinaryInteger.read(data, cur, bytes) + def read_int(bytes: Int) : Long = { + val num = BinaryInteger.readLong(data, cur, bytes) cur += bytes num } From 664899b52a745e9567f508222f3787e099d6a770 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 2 Mar 2016 17:43:01 +0100 Subject: [PATCH 63/75] version bump --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9ce4fdd..7d34ec1 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.1" +version := "0.8.2" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") From b94ae4c8cbbe4f13293062a87fd28f53a6e38aad Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 4 Mar 2016 10:34:15 +0100 Subject: [PATCH 64/75] improve logging on some exception that is bugging us --- Dockerfile | 2 +- build.sbt | 2 +- src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a8d65e..b0ac477 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ ENV SQLTAP_HTTP_PORT="3000" \ ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb -RUN apt-get update && apt-get install sbt +RUN sbt ADD project /usr/src/project/ ADD src /usr/src/src/ diff --git a/build.sbt b/build.sbt index 7d34ec1..72818ed 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.2" +version := "0.8.3_pre" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala index 56131bb..77aae01 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala @@ -177,6 +177,8 @@ class SQLConnection(pool: AbstractSQLConnectionPool) extends TimeoutCallback { if (cur_qry != null) cur_qry.error(err) + else + Logger.exception(err, false) state = SQL_STATE_CLOSE heartbeat.cancel() From 9d03f2fbfd10cafd9710e3c91acff69001fdf10c Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 1 Apr 2016 14:16:47 +0200 Subject: [PATCH 65/75] Fix statsd reporting --- Dockerfile | 3 ++- bootup.sh | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0ac477..1f8d451 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,8 @@ ENV SQLTAP_HTTP_PORT="3000" \ MEMCACHE_HOST="" \ MEMCACHE_PORT="11211" \ MEMCACHE_QUEUELEN="8192" \ - MEMCACHE_NUMCONNS="20" + MEMCACHE_NUMCONNS="20" \ + STATSD_PREFIX="sqltap" ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb diff --git a/bootup.sh b/bootup.sh index 60a88a3..cbc4ce4 100755 --- a/bootup.sh +++ b/bootup.sh @@ -70,11 +70,15 @@ fi opts="$opts ${SQLTAP_OPTS}" report_to_statsd() { + if [[ -z "$STATSD_HOST" ]] || [[ -z "$STATSD_PORT" ]]; then + return + fi + sleep 10 # give some time to start sqltap while sleep 1; do curl "http://localhost:${SQLTAP_HTTP_PORT}/stats" | \ sed -e 's/,/\n/g' | \ - sed -e 's/^[^"]*"//g' -e 's/": *"/:/g' -e 's/".*$//g' -e 's/^/myprefix./g' | \ - nc -u -w0 ${STATSD_HOST} ${STATSD_PORT} + sed -e 's/^[^"]*"//g' -e 's/": *"/:/g' -e 's/".*$//g' -e 's/^/'$STATSD_PREFIX'./g' -e 's/\.[^:.]*$//' -e 's/$/|c/' > \ + /dev/udp/${STATSD_HOST}/${STATSD_PORT} done } From 58eb60ee12e75444312e963d3cfad7df4edd6fbe Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 1 Apr 2016 14:45:21 +0200 Subject: [PATCH 66/75] Make curl quite --- bootup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootup.sh b/bootup.sh index cbc4ce4..fc5de41 100755 --- a/bootup.sh +++ b/bootup.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -ex +set -e SQLTAP_JARFILE="/usr/lib/sqltap.jar" @@ -75,7 +75,7 @@ report_to_statsd() { fi sleep 10 # give some time to start sqltap while sleep 1; do - curl "http://localhost:${SQLTAP_HTTP_PORT}/stats" | \ + curl -s "http://localhost:${SQLTAP_HTTP_PORT}/stats" | \ sed -e 's/,/\n/g' | \ sed -e 's/^[^"]*"//g' -e 's/": *"/:/g' -e 's/".*$//g' -e 's/^/'$STATSD_PREFIX'./g' -e 's/\.[^:.]*$//' -e 's/$/|c/' > \ /dev/udp/${STATSD_HOST}/${STATSD_PORT} From 934656ff50e9d67bfc74530822857fa75c89e38f Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 26 Jun 2017 14:04:19 +0200 Subject: [PATCH 67/75] Support password --- bootup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bootup.sh b/bootup.sh index fc5de41..990b072 100755 --- a/bootup.sh +++ b/bootup.sh @@ -50,6 +50,11 @@ opts="$opts -t ${SQLTAP_THREADS}" opts="$opts --mysql-host ${MYSQL_HOST}" opts="$opts --mysql-port ${MYSQL_PORT}" opts="$opts --mysql-user ${MYSQL_USER}" + +if [[ "x${MYSQL_PASSWORD}" != "x" ]]; then + opts="$opts --mysql-password ${MYSQL_PASSWORD}" +fi + opts="$opts --mysql-database ${MYSQL_DATABASE}" opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}" opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}" From 7a2093253f88f9f2e41fbdb4b57480820984071c Mon Sep 17 00:00:00 2001 From: Alexander Jahraus Date: Thu, 15 Mar 2018 10:27:47 +0100 Subject: [PATCH 68/75] Allow empty value for SCHEMA_URL (#11) Before this change it was not possible to override the `SQLTAP_SCHEMA` with a custom value for trying things out in development, because this script would download the schema from the dist server and overwrite it, or exit with an error on line 17 if SCHEMA_URL was set to an empty string. --- bootup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootup.sh b/bootup.sh index 990b072..ffefc52 100755 --- a/bootup.sh +++ b/bootup.sh @@ -3,8 +3,8 @@ set -e SQLTAP_JARFILE="/usr/lib/sqltap.jar" -if [[ "$SCHEMA_URL" != "" ]]; then - curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" +if [[ $SCHEMA_URL == http* ]]; then + curl -sL $SCHEMA_URL -o "${SQLTAP_SCHEMA}" fi require_arg() { From 4be61f2ee46c2defe7e12684e56bcccdf314bd8d Mon Sep 17 00:00:00 2001 From: Alexander Jahraus Date: Thu, 15 Mar 2018 10:29:05 +0100 Subject: [PATCH 69/75] Add closing single quote to NotFoundException (#12) --- src/main/scala/com/paulasmuth/sqltap/Exceptions.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala b/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala index 36313df..3480876 100644 --- a/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala +++ b/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala @@ -28,7 +28,7 @@ class NotFoundException(cur: Instruction = null) extends Exception { if (cur == null) "not found" else - "could not find record '" + - (if (cur.relation == null) "null" else cur.relation.name) + - (if (cur.record.has_id) "' with id #" + cur.record.id.toString else "") + "could not find record " + + (if (cur.relation == null) "null" else "'" + cur.relation.name + "'") + + (if (cur.record.has_id) " with id #" + cur.record.id.toString else "") } From 2b5f574829f13abb99c0e2eb91a971c2fb90a979 Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 15 Mar 2018 10:30:17 +0100 Subject: [PATCH 70/75] Bump version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 72818ed..560c0ad 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.3_pre" +version := "0.8.4" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") From e3fdb8fd1eeefefe94cb953937d4a4ba71560e13 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 25 May 2018 13:38:43 +0200 Subject: [PATCH 71/75] Calculate length for mediumblob aka mediumtext --- .../com/paulasmuth/sqltap/mysql/LengthEncodedString.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala index 9d0fafa..51ba770 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala @@ -31,6 +31,13 @@ object LengthEncodedString { offset += 4 } + else if ((data(0) & 0x000000ff) == 0xfa) { + length += (data(pos + 1) & 0x000000ff) + length += (data(pos + 2) & 0x000000ff) << 8 + length += (data(pos + 3) & 0x000000ff) << 16 + offset += 4 + } + else if ((data(0) & 0x000000ff) == 0xfe && data.size == 9) throw new SQLProtocolError("length encoded string too large!") From 5297a2d9acee1cd632d2163032adee8ffb30cef1 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 25 May 2018 13:41:48 +0200 Subject: [PATCH 72/75] Bump version 0.8.5 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 560c0ad..433e14a 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.4" +version := "0.8.5" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") From bddc6d70d1363661ddaff4a8711e476bf8427a30 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 25 May 2018 14:19:19 +0200 Subject: [PATCH 73/75] Fix type detection --- build.sbt | 2 +- src/main/scala/com/paulasmuth/sqltap/SQLTap.scala | 2 +- .../scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 433e14a..88dbeb7 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.5" +version := "0.8.6" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") diff --git a/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala b/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala index b575d07..4227dbd 100644 --- a/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala @@ -19,7 +19,7 @@ import java.io.File object SQLTap{ - val VERSION = "v0.8.0" + val VERSION = "v0.8.6" def main(args: Array[String]) : Unit = { var n = 0 diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala index 51ba770..06821c2 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala @@ -31,7 +31,7 @@ object LengthEncodedString { offset += 4 } - else if ((data(0) & 0x000000ff) == 0xfa) { + else if ((data(pos) & 0x000000ff) == 0xfa) { length += (data(pos + 1) & 0x000000ff) length += (data(pos + 2) & 0x000000ff) << 8 length += (data(pos + 3) & 0x000000ff) << 16 From 98f5bb698caa7234ca5d10ca2aaac951f9075894 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 25 May 2018 15:01:02 +0200 Subject: [PATCH 74/75] Print type --- .../scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala index 06821c2..21f0939 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala @@ -42,7 +42,7 @@ object LengthEncodedString { throw new SQLProtocolError("length encoded string too large!") else - throw new SQLProtocolError("invalid length encoded string") + throw new SQLProtocolError("invalid length encoded string. pos: " + pos + ", type: " + (data(pos) & 0x000000ff)) val string = new String(data, offset, length, "UTF-8") From 3fdf594ee77d8e667f1628e3add8e84ff7e94865 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 25 May 2018 15:14:44 +0200 Subject: [PATCH 75/75] FIX blob type not detected --- .../paulasmuth/sqltap/mysql/LengthEncodedString.scala | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala index 21f0939..bfd753e 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala @@ -24,14 +24,7 @@ object LengthEncodedString { offset += 3 } - else if ((data(0) & 0x000000ff) == 0xfd) { - length += (data(pos + 1) & 0x000000ff) - length += (data(pos + 2) & 0x000000ff) << 8 - length += (data(pos + 3) & 0x000000ff) << 16 - offset += 4 - } - - else if ((data(pos) & 0x000000ff) == 0xfa) { + else if ((data(pos) & 0x000000ff) == 0xfd) { length += (data(pos + 1) & 0x000000ff) length += (data(pos + 2) & 0x000000ff) << 8 length += (data(pos + 3) & 0x000000ff) << 16