From 8bd413f3d7baa490e1b9406ace521dafb8a1b1dc Mon Sep 17 00:00:00 2001 From: Tom Corcoran Date: Wed, 26 Oct 2022 14:37:52 +0100 Subject: [PATCH 1/2] Updated to work with MySql v8 & improved help --- README.md | 17 ++++++++++------- pom.xml | 2 +- .../java/com/geeksumm/sqltojson/DBHelper.java | 2 +- .../java/com/geeksumm/sqltojson/SqlToJson.java | 10 +++++----- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e252d12..894a1a5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SqlToJson #### Mapping SQL Query result into JSON -This is tools to convert SQL Query result into JSON file. Implemented using JDBC and Currently only support MySQL, altough other DBMS is planned soon. +This is a tool to convert SQL Query results into JSON file. Implemented using JDBC and currently only supports MySQL. ## Usage 1. Import this project into Eclipse @@ -135,17 +135,20 @@ JSON Output : "to_date" : [1989-10-14, 1990-10-14, 1991-10-14, 1992-10-13, 1993-10-13, 1994-10-13, 1995-10-13, 1996-10-12, 1997-10-12, 1998-10-12, 1999-10-12, 2000-10-11, 2001-10-11, 9999-01-01] - -* Beware of the same collumn name between different table!. While JSON allow us to write duplicate field, only the last one will be used by other programs that consume the JSON. This probably isn't what you want +* The subnode startColumn and endColumn should correspond to the first and last column of each table in the combined view +* Beware of the same column name between different table!. While JSON allow us to write duplicate field, only the last one will be used by other programs that consume the JSON. This probably isn't what you want * If you use alias in your query, make sure the first column's alias is the same as docRoot * You *should* use alias if you joined table -* If your ResultSet contain multiple occurance of DocRoot (perhaps as FK), this tool is clever enough to ignore it. +* If your ResultSet contain multiple occurrence of DocRoot (perhaps as FK), this tool is clever enough to ignore it. ## Limitations * Data types limitations: * If you have column that use LONGVARCHAR to store multi-megabyte strings. It can be unwieldy * SQL Types ARRAY, BLOB, DISTINCT, CLOB, STRUCT, REF, and JAVA_OBJECT is ignored * Currently only support one level of nested nodes. +* There is an issue if the join clause field for one of the tables is not the same as the others +* The join clause field should be the first column in each table +* You can only exclude the join clause field, so the startColumn of one subNode should only be +2 more than the endColumn of the previous subNode ## Implementation Notes Since it's possible that results set is quite large, and the OEM Tree can't be fitted all in the memory, we'll be using Producer - Consumer Design Pattern approach. The producer is OEMTree object, that will read the ResultSet, build the tree, and put it into the queue. The consumer is WriteJSON object that will retrieve the value from queue, perform necessary cleaning, and write it into the JSON file. @@ -154,9 +157,9 @@ The queue implementation is [LinkedBlockingQueue](http://docs.oracle.com/javase/ You know from the samples that Joined Table have a lot of duplicate values. How did this tools map it up into clean, self-contained JSON? -1. Get Primary key for each collumn, and find its location in the ResultSet -2. Since Joined table have alot of duplicate / null value, we need to add a bit more cleaning before adding it to the branch. We'll use each collumn PK information to define wheter this cell's data is unique or not. -3. Finally, before writting OEM into JSON, we'll delete the duplicate by finding the minimum occurance of duplicate value in the branch. Then, cut the branch using sublist(0, branch.size()/duplicates) +1. Get Primary key for each column, and find its location in the ResultSet +2. Since Joined table have a lot of duplicate / null value, we need to add a bit more cleaning before adding it to the branch. We'll use each column PK information to define whether this cell's data is unique or not. +3. Finally, before witting OEM into JSON, we'll delete the duplicate by finding the minimum occurrence of duplicate value in the branch. Then, cut the branch using sublist(0, branch.size()/duplicates) ## To Do * More output stream choices diff --git a/pom.xml b/pom.xml index b4cca77..75faec8 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ mysql mysql-connector-java - 5.1.24 + 8.0.30 diff --git a/src/main/java/com/geeksumm/sqltojson/DBHelper.java b/src/main/java/com/geeksumm/sqltojson/DBHelper.java index a567d1e..a1fd297 100644 --- a/src/main/java/com/geeksumm/sqltojson/DBHelper.java +++ b/src/main/java/com/geeksumm/sqltojson/DBHelper.java @@ -49,7 +49,7 @@ public DBHelper(Config config) throws SQLException { switch (config.getJdbcDriver().toString()) { case "mysql": case "mariadb": - JDBC_DRIVER = "com.mysql.jdbc.Driver"; + JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; url = "jdbc:mysql://"; break; case "postgresql": diff --git a/src/main/java/com/geeksumm/sqltojson/SqlToJson.java b/src/main/java/com/geeksumm/sqltojson/SqlToJson.java index 0e808e0..a1006a4 100644 --- a/src/main/java/com/geeksumm/sqltojson/SqlToJson.java +++ b/src/main/java/com/geeksumm/sqltojson/SqlToJson.java @@ -2,13 +2,13 @@ /** * */ -import java.io.File; + import java.io.IOException; import java.sql.SQLException; -import java.util.*; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; - import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -34,7 +34,7 @@ public static void main(String[] args) { DBHelper database = null; try { - config = reader.readValue(new File("json/config.json"), Config.class); + config = reader.readValue(ClassLoader.getSystemClassLoader().getResourceAsStream("json/config.json"), Config.class); database = new DBHelper(config); } catch (SQLException e) { e.printStackTrace(); @@ -47,7 +47,7 @@ public static void main(String[] args) { System.exit(1); e.printStackTrace(); } catch (IOException e) { - System.out.println("File Not Found. Exitting.."); + System.out.println("File Not Found. Exiting.."); System.exit(1); e.printStackTrace(); } From fda681d62deb48cd7ea70b74a50262d7c4d89fd7 Mon Sep 17 00:00:00 2001 From: Tom Corcoran Date: Wed, 26 Oct 2022 15:02:37 +0100 Subject: [PATCH 2/2] Updated to work with MySql v8 & improved help --- README.md | 17 +++++++---------- pom.xml | 2 +- .../java/com/geeksumm/sqltojson/DBHelper.java | 2 +- .../java/com/geeksumm/sqltojson/SqlToJson.java | 10 +++++----- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 894a1a5..e252d12 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SqlToJson #### Mapping SQL Query result into JSON -This is a tool to convert SQL Query results into JSON file. Implemented using JDBC and currently only supports MySQL. +This is tools to convert SQL Query result into JSON file. Implemented using JDBC and Currently only support MySQL, altough other DBMS is planned soon. ## Usage 1. Import this project into Eclipse @@ -135,20 +135,17 @@ JSON Output : "to_date" : [1989-10-14, 1990-10-14, 1991-10-14, 1992-10-13, 1993-10-13, 1994-10-13, 1995-10-13, 1996-10-12, 1997-10-12, 1998-10-12, 1999-10-12, 2000-10-11, 2001-10-11, 9999-01-01] -* The subnode startColumn and endColumn should correspond to the first and last column of each table in the combined view -* Beware of the same column name between different table!. While JSON allow us to write duplicate field, only the last one will be used by other programs that consume the JSON. This probably isn't what you want + +* Beware of the same collumn name between different table!. While JSON allow us to write duplicate field, only the last one will be used by other programs that consume the JSON. This probably isn't what you want * If you use alias in your query, make sure the first column's alias is the same as docRoot * You *should* use alias if you joined table -* If your ResultSet contain multiple occurrence of DocRoot (perhaps as FK), this tool is clever enough to ignore it. +* If your ResultSet contain multiple occurance of DocRoot (perhaps as FK), this tool is clever enough to ignore it. ## Limitations * Data types limitations: * If you have column that use LONGVARCHAR to store multi-megabyte strings. It can be unwieldy * SQL Types ARRAY, BLOB, DISTINCT, CLOB, STRUCT, REF, and JAVA_OBJECT is ignored * Currently only support one level of nested nodes. -* There is an issue if the join clause field for one of the tables is not the same as the others -* The join clause field should be the first column in each table -* You can only exclude the join clause field, so the startColumn of one subNode should only be +2 more than the endColumn of the previous subNode ## Implementation Notes Since it's possible that results set is quite large, and the OEM Tree can't be fitted all in the memory, we'll be using Producer - Consumer Design Pattern approach. The producer is OEMTree object, that will read the ResultSet, build the tree, and put it into the queue. The consumer is WriteJSON object that will retrieve the value from queue, perform necessary cleaning, and write it into the JSON file. @@ -157,9 +154,9 @@ The queue implementation is [LinkedBlockingQueue](http://docs.oracle.com/javase/ You know from the samples that Joined Table have a lot of duplicate values. How did this tools map it up into clean, self-contained JSON? -1. Get Primary key for each column, and find its location in the ResultSet -2. Since Joined table have a lot of duplicate / null value, we need to add a bit more cleaning before adding it to the branch. We'll use each column PK information to define whether this cell's data is unique or not. -3. Finally, before witting OEM into JSON, we'll delete the duplicate by finding the minimum occurrence of duplicate value in the branch. Then, cut the branch using sublist(0, branch.size()/duplicates) +1. Get Primary key for each collumn, and find its location in the ResultSet +2. Since Joined table have alot of duplicate / null value, we need to add a bit more cleaning before adding it to the branch. We'll use each collumn PK information to define wheter this cell's data is unique or not. +3. Finally, before writting OEM into JSON, we'll delete the duplicate by finding the minimum occurance of duplicate value in the branch. Then, cut the branch using sublist(0, branch.size()/duplicates) ## To Do * More output stream choices diff --git a/pom.xml b/pom.xml index 75faec8..b4cca77 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ mysql mysql-connector-java - 8.0.30 + 5.1.24 diff --git a/src/main/java/com/geeksumm/sqltojson/DBHelper.java b/src/main/java/com/geeksumm/sqltojson/DBHelper.java index a1fd297..a567d1e 100644 --- a/src/main/java/com/geeksumm/sqltojson/DBHelper.java +++ b/src/main/java/com/geeksumm/sqltojson/DBHelper.java @@ -49,7 +49,7 @@ public DBHelper(Config config) throws SQLException { switch (config.getJdbcDriver().toString()) { case "mysql": case "mariadb": - JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; + JDBC_DRIVER = "com.mysql.jdbc.Driver"; url = "jdbc:mysql://"; break; case "postgresql": diff --git a/src/main/java/com/geeksumm/sqltojson/SqlToJson.java b/src/main/java/com/geeksumm/sqltojson/SqlToJson.java index a1006a4..0e808e0 100644 --- a/src/main/java/com/geeksumm/sqltojson/SqlToJson.java +++ b/src/main/java/com/geeksumm/sqltojson/SqlToJson.java @@ -2,13 +2,13 @@ /** * */ - +import java.io.File; import java.io.IOException; import java.sql.SQLException; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; + import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -34,7 +34,7 @@ public static void main(String[] args) { DBHelper database = null; try { - config = reader.readValue(ClassLoader.getSystemClassLoader().getResourceAsStream("json/config.json"), Config.class); + config = reader.readValue(new File("json/config.json"), Config.class); database = new DBHelper(config); } catch (SQLException e) { e.printStackTrace(); @@ -47,7 +47,7 @@ public static void main(String[] args) { System.exit(1); e.printStackTrace(); } catch (IOException e) { - System.out.println("File Not Found. Exiting.."); + System.out.println("File Not Found. Exitting.."); System.exit(1); e.printStackTrace(); }