From 6d4bab52492c1eae89d23e79bd802e98fd7eacae Mon Sep 17 00:00:00 2001 From: luxu1-ms Date: Mon, 25 Oct 2021 00:05:28 -0700 Subject: [PATCH 1/3] add samples for auto generated cols --- .../computed-col-test.ipynb | 472 +++++++++++++++--- .../sql-table-generation.ipynb | 270 +++++++++- 2 files changed, 662 insertions(+), 80 deletions(-) diff --git a/samples/computed-col-test-cases/computed-col-test.ipynb b/samples/computed-col-test-cases/computed-col-test.ipynb index a6498627..30a496e3 100644 --- a/samples/computed-col-test-cases/computed-col-test.ipynb +++ b/samples/computed-col-test-cases/computed-col-test.ipynb @@ -13,6 +13,12 @@ "version": 3 }, "pygments_lexer": "python3" + }, + "extensions": { + "azuredatastudio": { + "version": 1, + "views": [] + } } }, "nbformat_minor": 2, @@ -24,16 +30,23 @@ "# spark = SparkSession.builder.getOrCreate()\r\n", "# sc.setLogLevel(\"INFO\")\r\n", "spark.sparkContext.setLogLevel('DEBUG')\r\n", - "data = [(1, \"2020\", \"01\"),\r\n", - " (2, \"2020\", \"02\"),\r\n", - " (3, \"2020\", \"03\")]\r\n", - "columns = [\"Id\", \"Year\", \"Month\"]\r\n", - "df = spark.createDataFrame(data = data, schema = columns)\r\n", - "df.show()" + "spark.conf.set('spark.sql.caseSensitive', True)\r\n", + "\r\n", + "servername = \"jdbc:sqlserver://master-0.master-svc\"\r\n", + "dbname = \"connector_test_db\"\r\n", + "url = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n", + "\r\n", + "user = \"connector_user1\"\r\n", + "password = \"password123!#\" # Please specify password here" ], "metadata": { "azdata_cell_guid": "fd51c6bb-ca56-47f7-b8ed-62baad1bd782", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -44,7 +57,7 @@ { "data": { "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
3application_1617815319900_0008pysparkidleLinkLink✔
" + "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
32application_1628887230618_0066pysparkidleLinkLink✔
" }, "metadata": {}, "output_type": "display_data" @@ -55,7 +68,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "38d9bc1016e047b9b974ccaab4f34ad8" + "model_id": "57594e6996e94f2bb9181dee19ddaafa" } }, "metadata": {}, @@ -72,16 +85,11 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "fec6769d00164cb5a00c6f72cf40cf5b" + "model_id": "49029d8d1d164fd0966752ecdf3aa4d5" } }, "metadata": {}, "output_type": "display_data" - }, - { - "name": "stdout", - "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+", - "output_type": "stream" } ], "execution_count": 2 @@ -89,15 +97,30 @@ { "cell_type": "code", "source": [ - "servername = \"jdbc:sqlserver://master-0.master-svc\"\r\n", - "dbname = \"connector_test_db1\"\r\n", - "url = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n", + "data = [(1, \"2020\", \"01\"),\r\n", + " (2, \"2020\", \"02\"),\r\n", + " (3, \"2020\", \"03\")]\r\n", + "columns = [\"Id\", \"Year\", \"Month\"]\r\n", + "df = spark.createDataFrame(data = data, schema = columns)\r\n", + "df.show()\r\n", "\r\n", - "user = \"connector_user1\"\r\n", - "password = \"password123!#\" # Please specify password here" + "posts_data = [(1,'Intro','Hi There This is ABC'),\r\n", + " (2,'Intro','Hello I''m PQR'),\r\n", + " (3,'Re: Intro','Hey PQR This is XYZ'),\r\n", + " (4,'Geography','Im George from USA'),\r\n", + " (5,'Re:Geography','I''m Mary from OZ'),\r\n", + " (6,'Re:Geography','I''m Peter from UK')]\r\n", + "posts_columns = [\"PostID\", \"PostTitle\", \"PostBody\"]\r\n", + "posts_df = spark.createDataFrame(data = posts_data, schema = posts_columns)\r\n", + "posts_df.show()" ], "metadata": { - "azdata_cell_guid": "fdc8b9c8-36eb-4280-b32e-6a95b0e11e45" + "azdata_cell_guid": "fdc8b9c8-36eb-4280-b32e-6a95b0e11e45", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -106,21 +129,36 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "802063b9be4e4abf97c8fc2b84789b32" + "model_id": "b0b0e4f5ed664b71aae1a27adc41ddac" } }, "metadata": {}, "output_type": "display_data" + }, + { + "name": "stdout", + "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+", + "output_type": "stream" } ], - "execution_count": 3 + "execution_count": 5 + }, + { + "cell_type": "markdown", + "source": [ + "### **1\\. Write to tables with computed columns**" + ], + "metadata": { + "azdata_cell_guid": "9bd52c1c-fe0c-4f4c-8726-023f3ff642a6" + }, + "attachments": {} }, { "cell_type": "code", "source": [ "## append with computed columns at start and end\r\n", "# Write from Spark to SQL table using MSSQL Spark Connector\r\n", - "dbtable = \"test1\"\r\n", + "dbtable = \"test_computed_col_1\"\r\n", "\r\n", "try:\r\n", " df.write \\\r\n", @@ -130,6 +168,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", + " .option(\"schemaCheckEnabled\", False) \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -137,7 +176,12 @@ "print(\"MSSQL Connector write(append) succeeded \")" ], "metadata": { - "azdata_cell_guid": "2f936a04-6110-47d7-ab25-87308576ab57" + "azdata_cell_guid": "2f936a04-6110-47d7-ab25-87308576ab57", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -146,7 +190,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "4ebad3e502dd48a2924e8762da7de964" + "model_id": "ef11011fce0a4d86b2ab67d3506ca6d0" } }, "metadata": {}, @@ -158,13 +202,13 @@ "output_type": "stream" } ], - "execution_count": 25 + "execution_count": 6 }, { "cell_type": "code", "source": [ "#Read from SQL table using MSSQ Connector\r\n", - "dbtable = \"test1\"\r\n", + "dbtable = \"test_computed_col_1\"\r\n", "\r\n", "jdbcDF = spark.read \\\r\n", " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", @@ -176,7 +220,12 @@ "jdbcDF.show(5)" ], "metadata": { - "azdata_cell_guid": "f957925e-2d9a-4cf0-a610-28fcf1e746ba" + "azdata_cell_guid": "f957925e-2d9a-4cf0-a610-28fcf1e746ba", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -185,7 +234,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "2fd8cdda3c284875944c0399527cd36a" + "model_id": "3d68cffa23f74740b5c234aec7036761" } }, "metadata": {}, @@ -193,18 +242,18 @@ }, { "name": "stdout", - "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-01| 1|2020| 01|20202020|\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n+-------+---+----+-----+--------+", + "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+", "output_type": "stream" } ], - "execution_count": 26 + "execution_count": 7 }, { "cell_type": "code", "source": [ "## append with 1 computed column in between\r\n", "# Write from Spark to SQL table using MSSQL Spark Connector\r\n", - "dbtable = \"test2\"\r\n", + "dbtable = \"test_computed_col_2\"\r\n", "\r\n", "try:\r\n", " df.write \\\r\n", @@ -214,6 +263,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", + " .option(\"schemaCheckEnabled\", False) \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -222,7 +272,12 @@ ], "metadata": { "azdata_cell_guid": "f51da718-8527-45c0-a739-b0e64af4e7a6", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -231,7 +286,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "aa66d89ae91e464989173d8c25012168" + "model_id": "d0f1fb3556824b68acf73d1ef0cc4d6b" } }, "metadata": {}, @@ -243,13 +298,13 @@ "output_type": "stream" } ], - "execution_count": 23 + "execution_count": 8 }, { "cell_type": "code", "source": [ "#Read from SQL table using MSSQL Connector\r\n", - "dbtable = \"test2\"\r\n", + "dbtable = \"test_computed_col_2\"\r\n", "\r\n", "jdbcDF = spark.read \\\r\n", " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", @@ -261,7 +316,12 @@ "jdbcDF.show(5)" ], "metadata": { - "azdata_cell_guid": "bd1b3ccd-3ed7-4b21-9b57-95d1d774d76e" + "azdata_cell_guid": "bd1b3ccd-3ed7-4b21-9b57-95d1d774d76e", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -270,7 +330,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "3ce2396102ee419192cd8f41c06ea044" + "model_id": "83f7f1511dd14c26a6b63053fac7a2d3" } }, "metadata": {}, @@ -282,14 +342,14 @@ "output_type": "stream" } ], - "execution_count": 24 + "execution_count": 9 }, { "cell_type": "code", "source": [ "## append with 2 computed columns in between\r\n", "# Write from Spark to SQL table using MSSQL Spark Connector\r\n", - "dbtable = \"test3\"\r\n", + "dbtable = \"test_computed_col_3\"\r\n", "\r\n", "try:\r\n", " df.write \\\r\n", @@ -308,7 +368,12 @@ ], "metadata": { "azdata_cell_guid": "10bbf920-e50f-4098-be1b-f06024a52fac", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -317,7 +382,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "2967ac7f14e34d27970e0833b7eb5645" + "model_id": "880a7b25e6fc472dac6ca47f737ffdda" } }, "metadata": {}, @@ -329,13 +394,13 @@ "output_type": "stream" } ], - "execution_count": 21 + "execution_count": 10 }, { "cell_type": "code", "source": [ "#Read from SQL table using MSSQL Connector\r\n", - "dbtable = \"test3\"\r\n", + "dbtable = \"test_computed_col_3\"\r\n", "\r\n", "jdbcDF = spark.read \\\r\n", " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", @@ -348,7 +413,12 @@ ], "metadata": { "azdata_cell_guid": "1c81ee7d-62bc-4a4d-acfa-900b6efeb6ec", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -357,7 +427,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "c610b609efaa4d27b53e1a48a417888e" + "model_id": "94f5c08789b849ffa6e7469d0ccb9e34" } }, "metadata": {}, @@ -365,11 +435,11 @@ }, { "name": "stdout", - "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 1|2020-01|2020|20202020| 01|\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n+---+-------+----+--------+-----+", + "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+", "output_type": "stream" } ], - "execution_count": 22 + "execution_count": 11 }, { "cell_type": "code", @@ -377,7 +447,7 @@ "## append with 1 computed column in df and table, but table has 1 less col\r\n", "# set schemaCheckEnabled\" as False\r\n", "# Write from Spark to SQL table using MSSQL Spark Connector\r\n", - "dbtable = \"test4\"\r\n", + "dbtable = \"test_computed_col_4\"\r\n", "\r\n", "try:\r\n", " df.write \\\r\n", @@ -395,7 +465,12 @@ "print(\"MSSQL Connector write(append) done \")" ], "metadata": { - "azdata_cell_guid": "0fcd38ab-a367-45e7-8d80-acb978176c82" + "azdata_cell_guid": "0fcd38ab-a367-45e7-8d80-acb978176c82", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -404,7 +479,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "98bb7592e1b24920909dd4ed6542737a" + "model_id": "2bc5b9342d5a4ae8ba5a4f030aedc00e" } }, "metadata": {}, @@ -416,13 +491,13 @@ "output_type": "stream" } ], - "execution_count": 19 + "execution_count": 12 }, { "cell_type": "code", "source": [ "#Read from SQL table using MSSQL Connector\r\n", - "dbtable = \"test4\"\r\n", + "dbtable = \"test_computed_col_4\"\r\n", "\r\n", "jdbcDF = spark.read \\\r\n", " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", @@ -434,7 +509,12 @@ "jdbcDF.show(5)" ], "metadata": { - "azdata_cell_guid": "bb9add17-d962-4e9d-b407-cf1d43cbaf98" + "azdata_cell_guid": "bb9add17-d962-4e9d-b407-cf1d43cbaf98", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -443,7 +523,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "f7300341de024a018ddad006b91624fc" + "model_id": "26d3b2fe1d1e488a9304c40be6991d14" } }, "metadata": {}, @@ -455,7 +535,287 @@ "output_type": "stream" } ], + "execution_count": 13 + }, + { + "cell_type": "markdown", + "source": [ + "### **2\\. Write to Temporal Tables (Generated Always)**" + ], + "metadata": { + "azdata_cell_guid": "1ce0f440-3209-4ee2-9bb0-0fa82f0c55bb" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# temporal table: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"dbo.tempTest\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")\r\n", + "" + ], + "metadata": { + "azdata_cell_guid": "fa16ca86-1837-4613-9650-78eed59eef3e" + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "451e02a8fa0d43cabe0032632b296546" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "MSSQL Connector write(append) succeeded", + "output_type": "stream" + } + ], + "execution_count": 15 + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"dbo.tempTest\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(truncate = False)" + ], + "metadata": { + "azdata_cell_guid": "dcaa3a1b-4770-4a95-80ea-d3e193091ca0" + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "31ec9191e04341a28fb8654c4bf726da" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+", + "output_type": "stream" + } + ], + "execution_count": 16 + }, + { + "cell_type": "code", + "source": [ + "# temporal table: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"test_computed_temp\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "1283627e-074c-4ff5-8752-b4cdf88b27f3" + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "790a82624c2e4c6681debdc7db2efc1e" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "MSSQL Connector write(append) succeeded", + "output_type": "stream" + } + ], "execution_count": 20 + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"test_computed_temp\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(truncate = False)" + ], + "metadata": { + "azdata_cell_guid": "982598ad-2c64-4ee7-9ddf-6e1ae6f0d2c5" + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "87ba143688b246ffb9405b65d2cdb7a8" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+", + "output_type": "stream" + } + ], + "execution_count": 21 + }, + { + "cell_type": "markdown", + "source": [ + "### **3\\. Write to Graph Table**" + ], + "metadata": { + "azdata_cell_guid": "6ccbf927-2f99-44a1-a6d6-21510dbde904" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# Graph table: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"dbo.ForumPosts\"\r\n", + "\r\n", + "try:\r\n", + " posts_df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "1469a88a-58b9-4007-8616-e9c594176b1d" + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "1520e2d0d8294ce992442d4507a0479f" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "MSSQL Connector write(append) succeeded", + "output_type": "stream" + } + ], + "execution_count": 23 + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"dbo.ForumPosts\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(5)" + ], + "metadata": { + "azdata_cell_guid": "3095ba6d-e02f-4a0a-95d1-3e13be55fe20", + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "a244c97f89df4bb38f6db0e77e3d64ed" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows", + "output_type": "stream" + } + ], + "execution_count": 24 } ] } \ No newline at end of file diff --git a/samples/computed-col-test-cases/sql-table-generation.ipynb b/samples/computed-col-test-cases/sql-table-generation.ipynb index ae1abfa9..052e9b95 100644 --- a/samples/computed-col-test-cases/sql-table-generation.ipynb +++ b/samples/computed-col-test-cases/sql-table-generation.ipynb @@ -8,6 +8,12 @@ "language_info": { "name": "sql", "version": "" + }, + "extensions": { + "azuredatastudio": { + "version": 1, + "views": [] + } } }, "nbformat_minor": 2, @@ -16,10 +22,15 @@ { "cell_type": "code", "source": [ - "CREATE DATABASE connector_test_db1" + "CREATE DATABASE connector_test_db" ], "metadata": { - "azdata_cell_guid": "5b1fd0a6-20b8-4e6a-9578-10bea2a3a43d" + "azdata_cell_guid": "5b1fd0a6-20b8-4e6a-9578-10bea2a3a43d", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -42,7 +53,7 @@ { "cell_type": "code", "source": [ - "Use connector_test_db1;\r\n", + "Use connector_test_db;\r\n", "CREATE LOGIN connector_user1 WITH PASSWORD ='password123!#' \r\n", "CREATE USER connector_user1 FROM LOGIN connector_user1\r\n", "\r\n", @@ -62,7 +73,12 @@ "ALTER ROLE [db_datawriter] ADD MEMBER connector_user1" ], "metadata": { - "azdata_cell_guid": "378bb099-8499-4661-9745-527a1ed19579" + "azdata_cell_guid": "378bb099-8499-4661-9745-527a1ed19579", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -82,13 +98,23 @@ ], "execution_count": 2 }, + { + "cell_type": "markdown", + "source": [ + "### **1\\. Tables with computed columns**" + ], + "metadata": { + "azdata_cell_guid": "70883f0f-b454-4c6c-957f-272397b77029" + }, + "attachments": {} + }, { "cell_type": "code", "source": [ "-- append with computed columns at start and end\r\n", - "USE connector_test_db1;\r\n", + "USE connector_test_db;\r\n", "\r\n", - "CREATE TABLE test1\r\n", + "CREATE TABLE test_computed_col_1\r\n", "(\r\n", " Date As (Year + '-' + Month),\r\n", " Id bigint,\r\n", @@ -98,7 +124,12 @@ ")" ], "metadata": { - "azdata_cell_guid": "dde73601-c253-45c8-b385-94da828ef6da" + "azdata_cell_guid": "dde73601-c253-45c8-b385-94da828ef6da", + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -111,19 +142,19 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.090" + "text/html": "Total execution time: 00:00:00.011" }, "metadata": {} } ], - "execution_count": 10 + "execution_count": 1 }, { "cell_type": "code", "source": [ "-- append with 1 computed column in between\r\n", - "USE connector_test_db1;\r\n", - "CREATE TABLE test2\r\n", + "USE connector_test_db;\r\n", + "CREATE TABLE test_computed_col_2\r\n", "(\r\n", " Id bigint,\r\n", " Date As (Year + '-' + Month),\r\n", @@ -133,7 +164,12 @@ ], "metadata": { "azdata_cell_guid": "d25fe7a7-c8cf-498e-89f9-3335277a2db1", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -146,20 +182,20 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.103" + "text/html": "Total execution time: 00:00:00.010" }, "metadata": {} } ], - "execution_count": 11 + "execution_count": 2 }, { "cell_type": "code", "source": [ "-- append with 2 computed columns in between\r\n", - "USE connector_test_db1;\r\n", + "USE connector_test_db;\r\n", "\r\n", - "CREATE TABLE test3\r\n", + "CREATE TABLE test_computed_col_3\r\n", "(\r\n", " Id bigint,\r\n", " Date As (Year + '-' + Month),\r\n", @@ -170,7 +206,12 @@ ], "metadata": { "azdata_cell_guid": "2b952d30-7ca9-49d4-9cd8-92a35d30d881", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } }, "outputs": [ { @@ -183,20 +224,20 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.089" + "text/html": "Total execution time: 00:00:00.011" }, "metadata": {} } ], - "execution_count": 12 + "execution_count": 3 }, { "cell_type": "code", "source": [ "-- append with 1 computed column in df and table, but table has 1 less col\r\n", - "USE connector_test_db1;\r\n", + "USE connector_test_db;\r\n", "\r\n", - "CREATE TABLE test4\r\n", + "CREATE TABLE test_computed_col_4\r\n", "(\r\n", " Id bigint,\r\n", " Year nvarchar(4),\r\n", @@ -205,7 +246,58 @@ ], "metadata": { "azdata_cell_guid": "e7b24025-05e0-40de-992f-0ca41bfcc246", - "tags": [] + "tags": [], + "extensions": { + "azuredatastudio": { + "views": [] + } + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.012" + }, + "metadata": {} + } + ], + "execution_count": 4 + }, + { + "cell_type": "markdown", + "source": [ + "### **2\\. Temporal Tables (Generated Always)**" + ], + "metadata": { + "azdata_cell_guid": "d3d3ead0-0b97-43c5-988c-c186bc16f39c" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db;\r\n", + "CREATE TABLE dbo.tempTest\r\n", + "(\r\n", + " Id bigint NOT NULL PRIMARY KEY CLUSTERED\r\n", + " , Year nvarchar(4)\r\n", + " , Month nvarchar(2)\r\n", + " , [ValidFrom] datetime2 GENERATED ALWAYS AS ROW START\r\n", + " , [ValidTo] datetime2 GENERATED ALWAYS AS ROW END\r\n", + " , PERIOD FOR SYSTEM_TIME (ValidFrom, ValidTo)\r\n", + " )\r\n", + "WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.tempTestHistory));" + ], + "metadata": { + "azdata_cell_guid": "881271ce-3847-4b82-93a5-6e5be1d577f3" }, "outputs": [ { @@ -218,12 +310,142 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.257" + "text/html": "Total execution time: 00:00:00.015" }, "metadata": {} } ], - "execution_count": 16 + "execution_count": 13 + }, + { + "cell_type": "code", + "source": [ + "-- create table with comptued cols and then convert it to temporal tables\r\n", + "USE connector_test_db;\r\n", + "CREATE TABLE test_computed_temp\r\n", + "(\r\n", + " Date As (Year + '-' + Month),\r\n", + " Id bigint PRIMARY KEY NOT NULL,\r\n", + " Year nvarchar(4),\r\n", + " Month nvarchar(2),\r\n", + " Years AS REPLICATE(Year, 2)\r\n", + ")\r\n", + "\r\n", + "-- Convert test_computed_col_1 to temporal tables\r\n", + "ALTER TABLE test_computed_temp\r\n", + " ADD \r\n", + " [ValidFrom] datetime2 GENERATED ALWAYS AS ROW START DEFAULT GETUTCDATE()\r\n", + " , [ValidTo] datetime2 GENERATED ALWAYS AS ROW END DEFAULT CONVERT(DATETIME2, '9999-12-31 23:59:59.9999999')\r\n", + " , PERIOD FOR SYSTEM_TIME (ValidFrom, ValidTo)\r\n", + "\r\n", + "ALTER TABLE test_computed_temp SET (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.test_computed_temp_history));" + ], + "metadata": { + "azdata_cell_guid": "e42ca950-3d69-4f91-9790-bec158e50f70" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.032" + }, + "metadata": {} + } + ], + "execution_count": 27 + }, + { + "cell_type": "code", + "source": [ + "-- Use below to drop temporal tables\r\n", + "/*\r\n", + "USE connector_test_db;\r\n", + "ALTER TABLE dbo.tempTest SET (SYSTEM_VERSIONING = OFF);\r\n", + "DROP TABLE dbo.tempTest;\r\n", + "DROP TABLE dbo.tempTestHistory;\r\n", + "*/\r\n", + "\r\n", + "/*\r\n", + "USE connector_test_db;\r\n", + "ALTER TABLE dbo.test_computed_temp SET (SYSTEM_VERSIONING = OFF);\r\n", + "DROP TABLE dbo.test_computed_temp;\r\n", + "DROP TABLE test_computed_temp_history;\r\n", + "*/" + ], + "metadata": { + "azdata_cell_guid": "261485c2-7076-49cb-aeb5-1e4048da1eb0" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": [ + "### **3\\. Graph Table**" + ], + "metadata": { + "azdata_cell_guid": "02655951-5f8e-4420-973b-f5545f4658f6" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db\r\n", + "GO\r\n", + "\r\n", + "CREATE TABLE [dbo].[ForumPosts](\r\n", + " [PostID] [bigint] NULL,\r\n", + " [PostTitle] [varchar](100) NULL,\r\n", + " [PostBody] [varchar](1000) NULL\r\n", + ") AS NODE\r\n", + "GO\r\n", + "\r\n", + "CREATE TABLE [dbo].[Replies]\r\n", + "AS EDGE\r\n", + "GO" + ], + "metadata": { + "azdata_cell_guid": "07283020-b4c9-4a8d-af98-c1be3301a413" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.027" + }, + "metadata": {} + } + ], + "execution_count": 11 } ] } \ No newline at end of file From 108a50ebc5bcf0c8f25b5f3ff6e24a32af7ca342 Mon Sep 17 00:00:00 2001 From: luxu1 Date: Tue, 14 Dec 2021 01:10:15 -0800 Subject: [PATCH 2/3] modify for new PR --- .../computed-col-test.ipynb | 274 +++++++++++++----- .../sql-table-generation.ipynb | 115 +++++++- 2 files changed, 298 insertions(+), 91 deletions(-) diff --git a/samples/computed-col-test-cases/computed-col-test.ipynb b/samples/computed-col-test-cases/computed-col-test.ipynb index 30a496e3..f612041b 100644 --- a/samples/computed-col-test-cases/computed-col-test.ipynb +++ b/samples/computed-col-test-cases/computed-col-test.ipynb @@ -50,19 +50,20 @@ }, "outputs": [ { + "output_type": "stream", "name": "stdout", - "text": "Starting Spark application\n", - "output_type": "stream" + "text": "Starting Spark application\n" }, { + "output_type": "display_data", "data": { "text/plain": "", "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
32application_1628887230618_0066pysparkidleLinkLink✔
" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -71,15 +72,15 @@ "model_id": "57594e6996e94f2bb9181dee19ddaafa" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "SparkSession available as 'spark'.\n", - "output_type": "stream" + "text": "SparkSession available as 'spark'.\n" }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -88,8 +89,7 @@ "model_id": "49029d8d1d164fd0966752ecdf3aa4d5" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} } ], "execution_count": 2 @@ -124,6 +124,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -132,13 +133,12 @@ "model_id": "b0b0e4f5ed664b71aae1a27adc41ddac" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+", - "output_type": "stream" + "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+" } ], "execution_count": 5 @@ -168,7 +168,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -185,6 +185,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -193,13 +194,12 @@ "model_id": "ef11011fce0a4d86b2ab67d3506ca6d0" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 6 @@ -229,6 +229,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -237,13 +238,12 @@ "model_id": "3d68cffa23f74740b5c234aec7036761" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+", - "output_type": "stream" + "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+" } ], "execution_count": 7 @@ -263,7 +263,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -281,6 +281,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -289,13 +290,12 @@ "model_id": "d0f1fb3556824b68acf73d1ef0cc4d6b" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 8 @@ -325,6 +325,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -333,13 +334,12 @@ "model_id": "83f7f1511dd14c26a6b63053fac7a2d3" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+", - "output_type": "stream" + "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+" } ], "execution_count": 9 @@ -359,7 +359,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -377,6 +377,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -385,13 +386,12 @@ "model_id": "880a7b25e6fc472dac6ca47f737ffdda" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 10 @@ -422,6 +422,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -430,13 +431,12 @@ "model_id": "94f5c08789b849ffa6e7469d0ccb9e34" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+", - "output_type": "stream" + "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+" } ], "execution_count": 11 @@ -457,7 +457,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -474,6 +474,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -482,13 +483,12 @@ "model_id": "2bc5b9342d5a4ae8ba5a4f030aedc00e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 12 @@ -518,6 +518,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -526,13 +527,12 @@ "model_id": "26d3b2fe1d1e488a9304c40be6991d14" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+", - "output_type": "stream" + "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+" } ], "execution_count": 13 @@ -563,7 +563,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -576,6 +576,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -584,13 +585,12 @@ "model_id": "451e02a8fa0d43cabe0032632b296546" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 15 @@ -615,6 +615,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -623,13 +624,12 @@ "model_id": "31ec9191e04341a28fb8654c4bf726da" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+" } ], "execution_count": 16 @@ -650,7 +650,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -662,6 +662,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -670,13 +671,12 @@ "model_id": "790a82624c2e4c6681debdc7db2efc1e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 20 @@ -701,6 +701,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -709,13 +710,12 @@ "model_id": "87ba143688b246ffb9405b65d2cdb7a8" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+" } ], "execution_count": 21 @@ -746,7 +746,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"PostID, PostTitle, PostBody\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -758,6 +758,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -766,13 +767,12 @@ "model_id": "1520e2d0d8294ce992442d4507a0479f" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 23 @@ -798,6 +798,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -806,16 +807,135 @@ "model_id": "a244c97f89df4bb38f6db0e77e3d64ed" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows", - "output_type": "stream" + "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows" } ], "execution_count": 24 + }, + { + "cell_type": "markdown", + "source": [ + "### **4\\. Write to Tables with Identity columns**" + ], + "metadata": { + "azdata_cell_guid": "3fd6f1f2-cac4-42c1-a3c0-118de963d6ac" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# auto increment columns: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"test_identity\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"columnsToWrite\", \"Year, Month\") \\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "99bf84bc-c6bb-43a9-aa6b-9f3387cbcae1" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"test_identity\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(5)" + ], + "metadata": { + "azdata_cell_guid": "d477c6a5-4e69-4006-8742-786b41fb4031" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": [ + "### **5\\. Write to Table skipping some non-nullable columns**" + ], + "metadata": { + "azdata_cell_guid": "340e9ce8-eabb-4cd5-95e9-ff5b3e6b5b46" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# skip non-nullable column: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"test_skip_cols\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year\") \\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "98ee8100-f43f-468b-af97-9024144f002c" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"test_skip_cols\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(5)" + ], + "metadata": { + "azdata_cell_guid": "d687eb8f-435d-4ba9-b396-9e756185c4ca" + }, + "outputs": [], + "execution_count": null } ] } \ No newline at end of file diff --git a/samples/computed-col-test-cases/sql-table-generation.ipynb b/samples/computed-col-test-cases/sql-table-generation.ipynb index 052e9b95..44d3fec7 100644 --- a/samples/computed-col-test-cases/sql-table-generation.ipynb +++ b/samples/computed-col-test-cases/sql-table-generation.ipynb @@ -43,7 +43,7 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.640" + "text/html": "Total execution time: 00:00:00.287" }, "metadata": {} } @@ -91,7 +91,7 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.108" + "text/html": "Total execution time: 00:00:00.022" }, "metadata": {} } @@ -142,12 +142,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.011" + "text/html": "Total execution time: 00:00:00.018" }, "metadata": {} } ], - "execution_count": 1 + "execution_count": 3 }, { "cell_type": "code", @@ -182,12 +182,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.010" + "text/html": "Total execution time: 00:00:00.017" }, "metadata": {} } ], - "execution_count": 2 + "execution_count": 4 }, { "cell_type": "code", @@ -224,12 +224,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.011" + "text/html": "Total execution time: 00:00:00.018" }, "metadata": {} } ], - "execution_count": 3 + "execution_count": 5 }, { "cell_type": "code", @@ -264,12 +264,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.012" + "text/html": "Total execution time: 00:00:00.015" }, "metadata": {} } ], - "execution_count": 4 + "execution_count": 6 }, { "cell_type": "markdown", @@ -310,12 +310,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.015" + "text/html": "Total execution time: 00:00:00.022" }, "metadata": {} } ], - "execution_count": 13 + "execution_count": 7 }, { "cell_type": "code", @@ -359,7 +359,7 @@ "metadata": {} } ], - "execution_count": 27 + "execution_count": 8 }, { "cell_type": "code", @@ -440,7 +440,94 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.027" + "text/html": "Total execution time: 00:00:00.046" + }, + "metadata": {} + } + ], + "execution_count": 9 + }, + { + "cell_type": "markdown", + "source": [ + "### **4\\. Table with Identity column**" + ], + "metadata": { + "azdata_cell_guid": "8af5cf11-1bec-4049-a002-0678e0075afb" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db\r\n", + "\r\n", + "CREATE TABLE test_identity\r\n", + "(\r\n", + " Id bigint IDENTITY(1,1),\r\n", + " Year nvarchar(4),\r\n", + " Month nvarchar(2),\r\n", + " Years AS REPLICATE(Year, 2)\r\n", + ")" + ], + "metadata": { + "azdata_cell_guid": "66493ab5-2e0a-4249-b214-49f027008723" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.043" + }, + "metadata": {} + } + ], + "execution_count": 12 + }, + { + "cell_type": "markdown", + "source": [ + "### **5\\. Insert to table skipping some columns**" + ], + "metadata": { + "azdata_cell_guid": "6d24c74f-e28b-4bc1-9d86-2ba0bcb37f0b" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db\r\n", + "\r\n", + "CREATE TABLE test_skip_cols\r\n", + "(\r\n", + " Id bigint,\r\n", + " Year nvarchar(4),\r\n", + " Month nvarchar(2)\r\n", + ")" + ], + "metadata": { + "azdata_cell_guid": "7496e3c4-d5d0-46cc-b8c4-2cfaa40f01ae" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.019" }, "metadata": {} } From 850512d2bebd63627ab783da2af92e46d640b803 Mon Sep 17 00:00:00 2001 From: luxu1-ms Date: Wed, 26 Oct 2022 09:30:34 -0700 Subject: [PATCH 3/3] minor update --- .../computed-col-test.ipynb | 245 +++++++++++++----- .../sql-table-generation.ipynb | 130 ++++++---- 2 files changed, 260 insertions(+), 115 deletions(-) diff --git a/samples/computed-col-test-cases/computed-col-test.ipynb b/samples/computed-col-test-cases/computed-col-test.ipynb index f612041b..2cb07ee2 100644 --- a/samples/computed-col-test-cases/computed-col-test.ipynb +++ b/samples/computed-col-test-cases/computed-col-test.ipynb @@ -46,7 +46,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -58,7 +59,7 @@ "output_type": "display_data", "data": { "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
32application_1628887230618_0066pysparkidleLinkLink✔
" + "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
6application_1663974487519_0013pysparkidleLinkLink✔
" }, "metadata": {} }, @@ -69,7 +70,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "57594e6996e94f2bb9181dee19ddaafa" + "model_id": "9cf85570cd994e8a8545e85a0aa2573b" } }, "metadata": {} @@ -86,7 +87,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "49029d8d1d164fd0966752ecdf3aa4d5" + "model_id": "0339dfbf3823410fb06a568e4816cb8a" } }, "metadata": {} @@ -97,13 +98,23 @@ { "cell_type": "code", "source": [ + "from pyspark.sql.types import StructType, StructField, StringType, IntegerType\r\n", + "\r\n", "data = [(1, \"2020\", \"01\"),\r\n", " (2, \"2020\", \"02\"),\r\n", " (3, \"2020\", \"03\")]\r\n", - "columns = [\"Id\", \"Year\", \"Month\"]\r\n", + "\r\n", + "columns = StructType([\r\n", + " StructField(\"Id\", IntegerType(), nullable = False),\r\n", + " StructField(\"Year\", StringType(), nullable = True),\r\n", + " StructField(\"Month\", StringType(), nullable = True)\r\n", + "])\r\n", + "\r\n", "df = spark.createDataFrame(data = data, schema = columns)\r\n", "df.show()\r\n", + "df.printSchema()\r\n", "\r\n", + "# posts_data dataframe is for Graph table\r\n", "posts_data = [(1,'Intro','Hi There This is ABC'),\r\n", " (2,'Intro','Hello I''m PQR'),\r\n", " (3,'Re: Intro','Hey PQR This is XYZ'),\r\n", @@ -120,7 +131,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -130,7 +142,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "b0b0e4f5ed664b71aae1a27adc41ddac" + "model_id": "e34c302c9dc24579ba6ba1dfbf2cb8ad" } }, "metadata": {} @@ -138,10 +150,10 @@ { "output_type": "stream", "name": "stdout", - "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+" + "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\nroot\n |-- Id: integer (nullable = false)\n |-- Year: string (nullable = true)\n |-- Month: string (nullable = true)\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+" } ], - "execution_count": 5 + "execution_count": 3 }, { "cell_type": "markdown", @@ -181,7 +193,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -191,7 +204,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "ef11011fce0a4d86b2ab67d3506ca6d0" + "model_id": "559afaa5815f4ed589d7824b5afc1037" } }, "metadata": {} @@ -202,7 +215,7 @@ "text": "MSSQL Connector write(append) succeeded" } ], - "execution_count": 6 + "execution_count": 5 }, { "cell_type": "code", @@ -225,7 +238,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -235,7 +249,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "3d68cffa23f74740b5c234aec7036761" + "model_id": "a96f82f316e34222af4e04d15beba91a" } }, "metadata": {} @@ -246,7 +260,7 @@ "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+" } ], - "execution_count": 7 + "execution_count": 6 }, { "cell_type": "code", @@ -277,7 +291,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -287,7 +302,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "d0f1fb3556824b68acf73d1ef0cc4d6b" + "model_id": "cb2c59ccf8f741958f3f986811fa6df1" } }, "metadata": {} @@ -298,7 +313,7 @@ "text": "MSSQL Connector write(append) succeeded" } ], - "execution_count": 8 + "execution_count": 7 }, { "cell_type": "code", @@ -321,7 +336,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -331,7 +347,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "83f7f1511dd14c26a6b63053fac7a2d3" + "model_id": "665800375fd94420b898b52ba2cfa4e3" } }, "metadata": {} @@ -342,7 +358,7 @@ "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+" } ], - "execution_count": 9 + "execution_count": 8 }, { "cell_type": "code", @@ -373,7 +389,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -383,7 +400,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "880a7b25e6fc472dac6ca47f737ffdda" + "model_id": "dccd78bd9d9e44468d1f0e017e4cfbe4" } }, "metadata": {} @@ -394,7 +411,7 @@ "text": "MSSQL Connector write(append) done" } ], - "execution_count": 10 + "execution_count": 9 }, { "cell_type": "code", @@ -418,7 +435,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -428,7 +446,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "94f5c08789b849ffa6e7469d0ccb9e34" + "model_id": "1e8b69a61aec4538bae15667f76e9976" } }, "metadata": {} @@ -436,16 +454,15 @@ { "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+" + "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 1|2020-01|2020|20202020| 01|\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n+---+-------+----+--------+-----+" } ], - "execution_count": 11 + "execution_count": 10 }, { "cell_type": "code", "source": [ "## append with 1 computed column in df and table, but table has 1 less col\r\n", - "# set schemaCheckEnabled\" as False\r\n", "# Write from Spark to SQL table using MSSQL Spark Connector\r\n", "dbtable = \"test_computed_col_4\"\r\n", "\r\n", @@ -470,7 +487,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -480,7 +498,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "2bc5b9342d5a4ae8ba5a4f030aedc00e" + "model_id": "616166dc39d842f4bee396b55934e857" } }, "metadata": {} @@ -491,7 +509,7 @@ "text": "MSSQL Connector write(append) done" } ], - "execution_count": 12 + "execution_count": 11 }, { "cell_type": "code", @@ -514,7 +532,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "python" }, "outputs": [ { @@ -524,7 +543,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "26d3b2fe1d1e488a9304c40be6991d14" + "model_id": "2fd506b5123d4e7cbedf6e04eb82e7cb" } }, "metadata": {} @@ -532,10 +551,10 @@ { "output_type": "stream", "name": "stdout", - "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+" + "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 2|2020|20202020|\n| 3|2020|20202020|\n| 1|2020|20202020|\n+---+----+--------+" } ], - "execution_count": 13 + "execution_count": 12 }, { "cell_type": "markdown", @@ -569,10 +588,12 @@ " print(\"MSSQL Connector write failed\", error)\r\n", "\r\n", "print(\"MSSQL Connector write(append) succeeded \")\r\n", - "" + "\r\n", + " #.option(\"schemaCheckEnabled\", False) \\" ], "metadata": { - "azdata_cell_guid": "fa16ca86-1837-4613-9650-78eed59eef3e" + "azdata_cell_guid": "fa16ca86-1837-4613-9650-78eed59eef3e", + "language": "python" }, "outputs": [ { @@ -582,7 +603,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "451e02a8fa0d43cabe0032632b296546" + "model_id": "c110870b4a7f4b60a7b13e19cb0ac37a" } }, "metadata": {} @@ -593,7 +614,7 @@ "text": "MSSQL Connector write(append) succeeded" } ], - "execution_count": 15 + "execution_count": 14 }, { "cell_type": "code", @@ -611,7 +632,8 @@ "jdbcDF.show(truncate = False)" ], "metadata": { - "azdata_cell_guid": "dcaa3a1b-4770-4a95-80ea-d3e193091ca0" + "azdata_cell_guid": "dcaa3a1b-4770-4a95-80ea-d3e193091ca0", + "language": "python" }, "outputs": [ { @@ -621,7 +643,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "31ec9191e04341a28fb8654c4bf726da" + "model_id": "e91c9d4d9a81439a927dd4cc8afd4310" } }, "metadata": {} @@ -629,15 +651,15 @@ { "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+" + "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2022-10-26 06:49:53.320574|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2022-10-26 06:49:53.332836|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2022-10-26 06:49:53.332836|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+" } ], - "execution_count": 16 + "execution_count": 15 }, { "cell_type": "code", "source": [ - "# temporal table: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "# temporal table with computed columns: Write from Spark to SQL table using MSSQL Spark Connector\r\n", "dbtable = \"test_computed_temp\"\r\n", "\r\n", "try:\r\n", @@ -651,6 +673,7 @@ " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", + " .option(\"schemaCheckEnabled\", False) \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -658,7 +681,8 @@ "print(\"MSSQL Connector write(append) succeeded \")" ], "metadata": { - "azdata_cell_guid": "1283627e-074c-4ff5-8752-b4cdf88b27f3" + "azdata_cell_guid": "1283627e-074c-4ff5-8752-b4cdf88b27f3", + "language": "python" }, "outputs": [ { @@ -668,7 +692,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "790a82624c2e4c6681debdc7db2efc1e" + "model_id": "3ef61c42d4d2462cba12b5cc87f5313a" } }, "metadata": {} @@ -679,7 +703,7 @@ "text": "MSSQL Connector write(append) succeeded" } ], - "execution_count": 20 + "execution_count": 16 }, { "cell_type": "code", @@ -697,7 +721,8 @@ "jdbcDF.show(truncate = False)" ], "metadata": { - "azdata_cell_guid": "982598ad-2c64-4ee7-9ddf-6e1ae6f0d2c5" + "azdata_cell_guid": "982598ad-2c64-4ee7-9ddf-6e1ae6f0d2c5", + "language": "python" }, "outputs": [ { @@ -707,7 +732,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "87ba143688b246ffb9405b65d2cdb7a8" + "model_id": "3089a3ff1367419d848523040b5ba510" } }, "metadata": {} @@ -715,10 +740,10 @@ { "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+" + "text": "read data from SQL server table \n+-------+---+----+-----+--------+--------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+--------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2022-10-26 06:50:00.130968|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2022-10-26 06:50:00.143209|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2022-10-26 06:50:00.143209|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+--------------------------+--------------------------+" } ], - "execution_count": 21 + "execution_count": 17 }, { "cell_type": "markdown", @@ -754,7 +779,8 @@ "print(\"MSSQL Connector write(append) succeeded \")" ], "metadata": { - "azdata_cell_guid": "1469a88a-58b9-4007-8616-e9c594176b1d" + "azdata_cell_guid": "1469a88a-58b9-4007-8616-e9c594176b1d", + "language": "python" }, "outputs": [ { @@ -764,7 +790,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "1520e2d0d8294ce992442d4507a0479f" + "model_id": "8b84231f2eef40128a412e07489114d1" } }, "metadata": {} @@ -775,7 +801,7 @@ "text": "MSSQL Connector write(append) succeeded" } ], - "execution_count": 23 + "execution_count": 18 }, { "cell_type": "code", @@ -794,7 +820,8 @@ ], "metadata": { "azdata_cell_guid": "3095ba6d-e02f-4a0a-95d1-3e13be55fe20", - "tags": [] + "tags": [], + "language": "python" }, "outputs": [ { @@ -804,7 +831,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "a244c97f89df4bb38f6db0e77e3d64ed" + "model_id": "4997b53c5a3448968645ff6ddfa02164" } }, "metadata": {} @@ -812,10 +839,10 @@ { "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows" + "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_1AEAB501C3FA43E098D64CFA11B7ABC8|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows" } ], - "execution_count": 24 + "execution_count": 19 }, { "cell_type": "markdown", @@ -851,10 +878,29 @@ "print(\"MSSQL Connector write(append) succeeded \")" ], "metadata": { - "azdata_cell_guid": "99bf84bc-c6bb-43a9-aa6b-9f3387cbcae1" + "azdata_cell_guid": "99bf84bc-c6bb-43a9-aa6b-9f3387cbcae1", + "language": "python" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "6c0b5dd47806439ea62b29299c0cd768" + } + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "MSSQL Connector write(append) succeeded" + } + ], + "execution_count": 20 }, { "cell_type": "code", @@ -872,10 +918,29 @@ "jdbcDF.show(5)" ], "metadata": { - "azdata_cell_guid": "d477c6a5-4e69-4006-8742-786b41fb4031" + "azdata_cell_guid": "d477c6a5-4e69-4006-8742-786b41fb4031", + "language": "python" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "4d41be9ff5714f4cb114d0d67708ce77" + } + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "read data from SQL server table \n+---+----+-----+--------+\n| Id|Year|Month| Years|\n+---+----+-----+--------+\n| 1|2020| 02|20202020|\n| 2|2020| 03|20202020|\n| 3|2020| 01|20202020|\n+---+----+-----+--------+" + } + ], + "execution_count": 21 }, { "cell_type": "markdown", @@ -911,10 +976,29 @@ "print(\"MSSQL Connector write(append) succeeded \")" ], "metadata": { - "azdata_cell_guid": "98ee8100-f43f-468b-af97-9024144f002c" + "azdata_cell_guid": "98ee8100-f43f-468b-af97-9024144f002c", + "language": "python" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "058856628d3f4f3facd048ff1f79f7a7" + } + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "MSSQL Connector write(append) succeeded" + } + ], + "execution_count": 24 }, { "cell_type": "code", @@ -932,10 +1016,29 @@ "jdbcDF.show(5)" ], "metadata": { - "azdata_cell_guid": "d687eb8f-435d-4ba9-b396-9e756185c4ca" + "azdata_cell_guid": "d687eb8f-435d-4ba9-b396-9e756185c4ca", + "language": "python" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "c2b18433ae964dd18f2b01969becc1d2" + } + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "read data from SQL server table \n+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 2|2020| null|\n| 3|2020| null|\n| 1|2020| null|\n+---+----+-----+" + } + ], + "execution_count": 25 } ] } \ No newline at end of file diff --git a/samples/computed-col-test-cases/sql-table-generation.ipynb b/samples/computed-col-test-cases/sql-table-generation.ipynb index 44d3fec7..6a2f0202 100644 --- a/samples/computed-col-test-cases/sql-table-generation.ipynb +++ b/samples/computed-col-test-cases/sql-table-generation.ipynb @@ -30,7 +30,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -43,12 +44,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.287" + "text/html": "Total execution time: 00:00:00.562" }, "metadata": {} } ], - "execution_count": 1 + "execution_count": 2 }, { "cell_type": "code", @@ -78,7 +79,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -91,12 +93,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.022" + "text/html": "Total execution time: 00:00:00.036" }, "metadata": {} } ], - "execution_count": 2 + "execution_count": 3 }, { "cell_type": "markdown", @@ -117,7 +119,7 @@ "CREATE TABLE test_computed_col_1\r\n", "(\r\n", " Date As (Year + '-' + Month),\r\n", - " Id bigint,\r\n", + " Id int not null,\r\n", " Year nvarchar(4),\r\n", " Month nvarchar(2),\r\n", " Years AS REPLICATE(Year, 2)\r\n", @@ -129,7 +131,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -142,12 +145,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.018" + "text/html": "Total execution time: 00:00:00.019" }, "metadata": {} } ], - "execution_count": 3 + "execution_count": 14 }, { "cell_type": "code", @@ -156,7 +159,7 @@ "USE connector_test_db;\r\n", "CREATE TABLE test_computed_col_2\r\n", "(\r\n", - " Id bigint,\r\n", + " Id int not null,\r\n", " Date As (Year + '-' + Month),\r\n", " Year nvarchar(4),\r\n", " Month nvarchar(2)\r\n", @@ -169,7 +172,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -182,12 +186,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.017" + "text/html": "Total execution time: 00:00:00.013" }, "metadata": {} } ], - "execution_count": 4 + "execution_count": 18 }, { "cell_type": "code", @@ -197,7 +201,7 @@ "\r\n", "CREATE TABLE test_computed_col_3\r\n", "(\r\n", - " Id bigint,\r\n", + " Id int not null,\r\n", " Date As (Year + '-' + Month),\r\n", " Year nvarchar(4),\r\n", " Years AS REPLICATE(Year, 2),\r\n", @@ -211,7 +215,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -224,12 +229,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.018" + "text/html": "Total execution time: 00:00:00.016" }, "metadata": {} } ], - "execution_count": 5 + "execution_count": 19 }, { "cell_type": "code", @@ -239,7 +244,7 @@ "\r\n", "CREATE TABLE test_computed_col_4\r\n", "(\r\n", - " Id bigint,\r\n", + " Id int not null,\r\n", " Year nvarchar(4),\r\n", " Years AS REPLICATE(Year, 2)\r\n", ")" @@ -251,7 +256,8 @@ "azuredatastudio": { "views": [] } - } + }, + "language": "sql" }, "outputs": [ { @@ -264,12 +270,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.015" + "text/html": "Total execution time: 00:00:00.013" }, "metadata": {} } ], - "execution_count": 6 + "execution_count": 20 }, { "cell_type": "markdown", @@ -287,7 +293,7 @@ "USE connector_test_db;\r\n", "CREATE TABLE dbo.tempTest\r\n", "(\r\n", - " Id bigint NOT NULL PRIMARY KEY CLUSTERED\r\n", + " Id int NOT NULL PRIMARY KEY CLUSTERED\r\n", " , Year nvarchar(4)\r\n", " , Month nvarchar(2)\r\n", " , [ValidFrom] datetime2 GENERATED ALWAYS AS ROW START\r\n", @@ -297,7 +303,8 @@ "WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.tempTestHistory));" ], "metadata": { - "azdata_cell_guid": "881271ce-3847-4b82-93a5-6e5be1d577f3" + "azdata_cell_guid": "881271ce-3847-4b82-93a5-6e5be1d577f3", + "language": "sql" }, "outputs": [ { @@ -310,12 +317,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.022" + "text/html": "Total execution time: 00:00:00.021" }, "metadata": {} } ], - "execution_count": 7 + "execution_count": 22 }, { "cell_type": "code", @@ -325,7 +332,7 @@ "CREATE TABLE test_computed_temp\r\n", "(\r\n", " Date As (Year + '-' + Month),\r\n", - " Id bigint PRIMARY KEY NOT NULL,\r\n", + " Id int PRIMARY KEY NOT NULL,\r\n", " Year nvarchar(4),\r\n", " Month nvarchar(2),\r\n", " Years AS REPLICATE(Year, 2)\r\n", @@ -341,7 +348,8 @@ "ALTER TABLE test_computed_temp SET (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.test_computed_temp_history));" ], "metadata": { - "azdata_cell_guid": "e42ca950-3d69-4f91-9790-bec158e50f70" + "azdata_cell_guid": "e42ca950-3d69-4f91-9790-bec158e50f70", + "language": "sql" }, "outputs": [ { @@ -354,12 +362,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.032" + "text/html": "Total execution time: 00:00:00.051" }, "metadata": {} } ], - "execution_count": 8 + "execution_count": 23 }, { "cell_type": "code", @@ -380,10 +388,26 @@ "*/" ], "metadata": { - "azdata_cell_guid": "261485c2-7076-49cb-aeb5-1e4048da1eb0" + "azdata_cell_guid": "261485c2-7076-49cb-aeb5-1e4048da1eb0", + "language": "sql" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.027" + }, + "metadata": {} + } + ], + "execution_count": 21 }, { "cell_type": "markdown", @@ -413,7 +437,8 @@ "GO" ], "metadata": { - "azdata_cell_guid": "07283020-b4c9-4a8d-af98-c1be3301a413" + "azdata_cell_guid": "07283020-b4c9-4a8d-af98-c1be3301a413", + "language": "sql" }, "outputs": [ { @@ -440,12 +465,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.046" + "text/html": "Total execution time: 00:00:00.037" }, "metadata": {} } ], - "execution_count": 9 + "execution_count": 10 }, { "cell_type": "markdown", @@ -464,14 +489,16 @@ "\r\n", "CREATE TABLE test_identity\r\n", "(\r\n", - " Id bigint IDENTITY(1,1),\r\n", + " Id int IDENTITY(1,1),\r\n", " Year nvarchar(4),\r\n", " Month nvarchar(2),\r\n", " Years AS REPLICATE(Year, 2)\r\n", ")" ], "metadata": { - "azdata_cell_guid": "66493ab5-2e0a-4249-b214-49f027008723" + "azdata_cell_guid": "66493ab5-2e0a-4249-b214-49f027008723", + "language": "sql", + "tags": [] }, "outputs": [ { @@ -484,12 +511,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.043" + "text/html": "Total execution time: 00:00:00.015" }, "metadata": {} } ], - "execution_count": 12 + "execution_count": 11 }, { "cell_type": "markdown", @@ -508,13 +535,14 @@ "\r\n", "CREATE TABLE test_skip_cols\r\n", "(\r\n", - " Id bigint,\r\n", + " Id int not null,\r\n", " Year nvarchar(4),\r\n", " Month nvarchar(2)\r\n", ")" ], "metadata": { - "azdata_cell_guid": "7496e3c4-d5d0-46cc-b8c4-2cfaa40f01ae" + "azdata_cell_guid": "7496e3c4-d5d0-46cc-b8c4-2cfaa40f01ae", + "language": "sql" }, "outputs": [ { @@ -527,12 +555,26 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.019" + "text/html": "Total execution time: 00:00:00.015" }, "metadata": {} } ], - "execution_count": 11 + "execution_count": 28 + }, + { + "cell_type": "code", + "source": [ + "DROP LOGIN connector_user1\r\n", + "DROP USER connector_user1\r\n", + "DROP DATABASE connector_test_db" + ], + "metadata": { + "language": "sql", + "azdata_cell_guid": "b3fbc40e-025c-4602-bcd5-41e19644ca5d" + }, + "outputs": [], + "execution_count": null } ] } \ No newline at end of file