good sql unit testing strategy

Author: bernatsort

.gitignore

@@ -2,9 +2,5 @@ __pycache__/
 .pytest_cache/
 archive/
 output.json
-data/validated_reference_dataset.xlsx
+#ata/validated_reference_dataset.xlsx
 
-# YAML validation rules instead of Python
-# rules_config.yaml
-# validation_rules_yaml.py
-# tests/test_sql_logic_yaml.py

data/validated_reference_dataset.xlsx

@@ -20,5 +20,10 @@
         "expected": [
             {"week": "2017-31", "active_patients": 30}
         ]
+    },
+    "active_patients_study_1368_0004": {
+        "expected": [
+            {"week": "2017-34", "active_patients": 60}
+        ]
     }
 }

data/validated_reference_dataset_errors.xlsx

@@ -1,6 +1,4 @@
 SELECT week, active_patients
 FROM clinical_trials
-WHERE week = "2017-31"
+WHERE week = "2017-31" AND study = "1368-0004"
 
--- Como hacer para que el test podamos generalizarlo para que sea active_patients en una fecha concreta. 
--- De esta forma podemos pasarle distintas fechas y mirar si en todas el test passa. 

expected_results/expected_results.json

@@ -0,0 +1,4 @@
+SELECT week, active_patients
+FROM clinical_trials
+WHERE week = "2017-XX" AND study = "1368-0004" 
+-- error introduced week = "2017-XX" should be week = "2017-34"

expected_results/expected_results_negative.json

@@ -14,15 +14,3 @@ def db_connection():
     connection = engine.connect()
     yield connection
     connection.close()
-
-@pytest.fixture(scope='module')
-def db_connection_negative():
-    engine = create_engine('sqlite:///:memory:')
-    file_path = os.path.join(os.path.dirname(__file__), '..', 'data', 'validated_reference_dataset_errors.xlsx')
-    df = pd.read_excel(file_path)
-    df.to_sql('clinical_trials', engine, if_exists='replace', index=False)
-
-    connection = engine.connect()
-    yield connection
-    connection.close()
-

rules_config.yaml

@@ -1,12 +1,8 @@
 # tests/test_sql_logic.py
 import sys
 import os
-# Add the parent directory to sys.path to locate validation_rules.py
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
 import pytest
 import json
-from validation_rules import validate_entry, common_rules
 
 @pytest.fixture(scope='module')
 def expected_results():
@@ -16,7 +12,7 @@ def fetch_results_as_dict(results):
     return [dict(row) for row in results]
 
 def load_expected_results():
-    json_path = os.path.join(os.path.dirname(__file__), '..', 'expected_results.json')
+    json_path = os.path.join(os.path.dirname(__file__), '..', 'expected_results/expected_results.json')
     with open(json_path) as f:
         return json.load(f)
 
@@ -36,20 +32,23 @@ def run_query_test(db_connection, expected_results, query_name, sql_file):
     results = db_connection.execute(query_sql)
     results_dict = fetch_results_as_dict(results)
 
-    # Filter valid results
-    filtered_results = [result for result in results_dict if validate_entry(result, common_rules)]
-
     # Compare with expected results
     expected = expected_results[query_name]["expected"]
-    assert filtered_results == expected, f"Error in {query_name}: {filtered_results} != {expected}"
+    assert results_dict == expected, f"Error in {query_name}: {results_dict} != {expected}"
 
+# Tests
 def test_active_sites_positive(db_connection, expected_results):
     run_query_test(db_connection, expected_results, "active_sites_positive", "active_sites_positive.sql")
 
 def test_randomized_patients_positive(db_connection, expected_results):
     run_query_test(db_connection, expected_results, "randomized_patients_positive", "randomized_patients_positive.sql")
 
+def test_active_patients_date(db_connection, expected_results):
+    run_query_test(db_connection, expected_results, "active_patients_date", "active_patients_date.sql")
 
+# Test that will fail
+def test_failing_active_patients_date(db_connection, expected_results):
+    run_query_test(db_connection, expected_results, "active_patients_study_1368_0004", "failing_query_active_patients_date.sql")
 
 # @pytest.mark.parametrize("query_name, sql_file", [
 #     ("active_sites_positive", "active_sites_positive.sql"),
@@ -70,52 +69,4 @@ def test_randomized_patients_positive(db_connection, expected_results):
 
 
 
-# import sys
-# import os
-# sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-# import pytest
-# import json
-# from validation_rules import validate_entry, common_rules
-
-# def fetch_results_as_dict(results):
-#     return [dict(row) for row in results]
-
-# def load_expected_results():
-#     json_path = os.path.join(os.path.dirname(__file__), '..', 'expected_results.json')
-#     with open(json_path) as f:
-#         return json.load(f)
-
-# expected_results = load_expected_results()
-
-# def load_sql_query(file_name):
-#     sql_path = os.path.join(os.path.dirname(__file__), '..', 'sql_queries', file_name)
-#     with open(sql_path, 'r') as file:
-#         return file.read()
-
-# def test_active_sites_positive(db_connection):
-#     query_name = "active_sites_positive"
-#     query_sql = load_sql_query("active_sites_positive.sql")
-#     results = db_connection.execute(query_sql)
-#     results_dict = fetch_results_as_dict(results)
-
-#     # Filtrar resultados válidos
-#     filtered_results = [result for result in results_dict if validate_entry(result, common_rules)]
-
-#     # Comparar con resultados esperados
-#     expected = expected_results[query_name]["expected"]
-#     assert filtered_results == expected, f"Error in {query_name}: {filtered_results} != {expected}"
-
-# def test_randomized_patients_positive(db_connection):
-#     query_name = "randomized_patients_positive"
-#     query_sql = load_sql_query("randomized_patients_positive.sql")
-#     results = db_connection.execute(query_sql)
-#     results_dict = fetch_results_as_dict(results)
-
-#     # Filtrar resultados válidos
-#     filtered_results = [result for result in results_dict if validate_entry(result, common_rules)]
-
-#     # Comparar con resultados esperados
-#     expected = expected_results[query_name]["expected"]
-#     assert filtered_results == expected, f"Error in {query_name}: {filtered_results} != {expected}"
-