diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 939f797..6e65916 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -229,7 +229,7 @@ 'k_survey_section': { 'reference_name': 'survey_section_reference', - 'col_list': ['namespace', 'surveyIdentifier', 'survey_section_title'], + 'col_list': ['namespace', 'surveyIdentifier', 'surveySectionTitle'], 'annualize': True }, @@ -342,7 +342,98 @@ 'col_list': [], 'annualize': False }, - + 'k_survey_section_response': { + 'reference_name': 'survey_section_response_reference', + 'col_list': ['surveyIdentifier', 'namespace', 'surveySectionTitle'], + 'annualize': True + }, + 'k_application': { + 'reference_name': 'application_reference', + 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'educationOrganizationId'], + 'annualize': True + }, + 'k_credential': { + 'reference_name': 'credential_reference', + 'col_list': ['credentialIdentifier', 'stateOfIssueStateAbbreviationDescriptor'], + 'annualize': True + }, + 'k_fieldwork_experience': { + 'reference_name': 'fieldwork_experience_reference', + 'col_list': ['beginDate', 'fieldworkIdentifier', 'studentUniqueId'], + 'annualize': True + }, + 'k_section': { + 'reference_name': 'section_reference', + 'col_list': ['localCourseCode', 'schoolId', 'schoolYear', 'sectionIdentifier', 'sessionName'], + 'annualize': False + }, + 'k_open_staff_position': { + 'reference_name': 'open_staff_position_reference', + 'col_list': ['educationOrganizationId', 'eventDate', 'openStaffPositionEventTypeDescriptor', 'requisitionNumber'], + 'annualize': True + }, + 'k_professional_development_event': { + 'reference_name': 'professional_development_event_reference', + 'col_list': ['namespace', 'professionalDevelopmentTitle'], + 'annualize': True + }, + 'k_evaluation_element_rating': { + 'reference_name': 'evaluation_element_rating_reference', + 'col_list': ['educationOrganizationId', + 'evaluationDate', + 'evaluationElementTitle', + 'evaluationObjectiveTitle', + 'evaluationPeriodDescriptor', + 'evaluationTitle', + 'performanceEvaluationTitle', + 'performanceEvaluationTypeDescriptor', + 'personId', + 'schoolYear', + 'sourceSystemDescriptor', + 'termDescriptor'], + 'annualize': True + }, + 'k_quantitative_measure': { + 'reference_name': 'quantitative_measure_reference', + 'col_list': [ + 'educationOrganizationId', + 'evaluationElementTitle', + 'evaluationObjectiveTitle', + 'evaluationPeriodDescriptor', + 'evaluationTitle', + 'performanceEvaluationTitle', + 'performanceEvaluationTypeDescriptor', + 'quantitative_measure_identifier', + 'schoolYear', + 'termDescriptor' + ], + 'annualize': True + }, + 'k_recruitment_event': { + 'reference_name': 'recruitment_event_reference', + 'col_list': ['educationOrganizationId', 'eventDate', 'eventTitle'], + 'annualize': True + }, + 'k_path': { + 'reference_name': 'path_reference', + 'col_list': ['educationOrganizationId', 'pathName'], + 'annualize': True + }, + 'k_student_path': { + 'reference_name': 'student_path_reference', + 'col_list': ['educationOrganizationId', 'pathName', 'studentUniqueId'], + 'annualize': True + }, + 'k_path_milestone': { + 'reference_name': 'path_milestone_reference', + 'col_list': ['educationOrganizationId', 'pathMilestoneName', 'pathMilestoneTypeDescriptor', 'pathName'], + 'annualize': True + }, + 'k_path_phase': { + 'reference_name': 'path_phase_reference', + 'col_list': ['educationOrganizationId', 'pathName', 'pathPhaseName'], + 'annualize': True + } } %} {#- retrieve key def for then decompose parts -#} diff --git a/models/staging/tpdm/base/_tpdm__base.yml b/models/staging/tpdm/base/_tpdm__base.yml index 81f41dc..651e838 100644 --- a/models/staging/tpdm/base/_tpdm__base.yml +++ b/models/staging/tpdm/base/_tpdm__base.yml @@ -86,3 +86,7 @@ models: tags: ['tpdm'] enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" + - name: base_tpdm__survey_section_response_person_target_associations + config: + tags: ['tpdm'] + enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" diff --git a/models/staging/tpdm/base/base_tpdm__application_events.sql b/models/staging/tpdm/base/base_tpdm__application_events.sql new file mode 100644 index 0000000..830f8d1 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__application_events.sql @@ -0,0 +1,36 @@ +with app_events as ( + {{ source_edfi3('application_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:sequenceNumber::int as sequence_number, + v:eventDate::date as event_date, -- this could also be event_start_date + {{ extract_descriptor('v:applicationEventTypeDescriptor::string') }} as application_event_type, + v:applicationReference:applicationIdentifier::string as application_identifier, + v:applicationReference:applicantProfileIdentifier::string as applicant_profile_identifier, + v:applicationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:eventEndDate::date as event_end_date + v:applicationEvaluationScore::float as application_evaluation_score + v:schoolYearTypeReference:schoolYear:int as school_year + -- references + v:applicationReference as application_reference + -- descriptors + {{ extract_descriptor('v:applicationEventResultDescriptor::string') }} as application_event_result_descriptor + {{ extract_descriptor('v:termDescriptor::string') }} as academic_term + -- edfi extensions + v:_ext as v_ext + + from app_events +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__credential_events.sql b/models/staging/tpdm/base/base_tpdm__credential_events.sql new file mode 100644 index 0000000..178785c --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__credential_events.sql @@ -0,0 +1,29 @@ +with app_events as ( + {{ source_edfi3('application_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:credentialEventDate::date as credential_event_date, + {{ extract_descriptor('v:credentialEventTypeDescriptor::string') }} as credential_event_type, + v:credentialReference:credentialIdentifier::string as credential_identifier, + {{ extract_descriptor('v:credentialReference:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue, + -- non-identity components + v:credentialEventReason::string as credential_event_reason + -- references + v:credentialReference as credential_reference + -- edfi extensions + v:_ext as v_ext + + from app_events +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql b/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql new file mode 100644 index 0000000..d12b569 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql @@ -0,0 +1,31 @@ +with student_section_associations as ( + {{ source_edfi3('student_section_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:fieldworkExperienceReference:beginDate::date as begin_date, + v:fieldworkExperienceReference:fieldworkIdentifier::string as fieldwork_identifier, + v:fieldworkExperienceReference:studentUniqueId::string as student_unique_id, + v:sectionReference:sectionIdentifier::string as section_identifier, + v:sectionReference:localCourseCode::string as local_course_code, + v:sectionReference:sessionName::string as session_name, + v:sectionReference:schoolId::int as school_id, + v:sectionReference:schoolYear::int as school_year, + -- references + v:fieldworkExperienceReference as fieldwork_experience_reference, + v:sectionReference as section_reference, + -- edfi extensions + v:_ext as v_ext + from student_section_associations +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__goals.sql b/models/staging/tpdm/base/base_tpdm__goals.sql new file mode 100644 index 0000000..9cdab37 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__goals.sql @@ -0,0 +1,37 @@ +with base_goals as ( + select * from {{ source_edfi3('goals') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:personReference:personId::string as person_id, + v:personReference:sourceSystemDescriptor::string as person_source_system, + v:goalTitle::string as goal_title, + v:assignmentDate::date as assignment_date, + -- non-identity components + v:goalDescription::string as goal_description, + v:comments::string as comments, + v:completed_indicator::boolean as is_completed, + v:completedDate::date as completed_date, + v:dueDate::date as due_date, + -- descriptors + {{ extract_descriptor('v:goalTypeDescriptor::string') }} as goal_type, + -- references + v:evaluationElementReference as evaluation_element_reference, + v:evaluationObjectiveReference as evaluation_objective_reference, + v:parentGoalReference as parent_goal_reference, + v:personReference as person_reference, + -- edfi extensions + v:_ext as v_ext + from base_goals +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql b/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql new file mode 100644 index 0000000..aff3b7d --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql @@ -0,0 +1,29 @@ +with open_staff_position_events as ( + {{ source_edfi3('open_staff_position_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:eventDate::date as event_date, + {{ extract_descriptor('v:openStaffPositionEventTypeDescriptor::string') }} as open_staff_position_event_type, -- name event_type instead? + v:openStaffPositionReference:educationOrganizationId::int as ed_org_id, + v:openStaffPositionReference:requisitionNumber::string as requisition_number, + -- descriptors + {{ extract_descriptor('v:openStaffPositionEventStatusDescriptor::string') }} as open_staff_position_event_status, -- name event_status instead? + -- references + v:openStaffPositionReference as open_staff_position_reference, + -- edfi extensions + v:_ext as v_ext + + from open_staff_position_events +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__path_milestones.sql b/models/staging/tpdm/base/base_tpdm__path_milestones.sql new file mode 100644 index 0000000..fa0c521 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__path_milestones.sql @@ -0,0 +1,26 @@ +with path_milestones as ( + {{ source_edfi3('path_milestones') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathMilestoneName::string as path_milestone_name, + {{ extract_descriptor('v:pathMilestoneTypeDescriptor::string') }} as path_milestone_type, + -- non-identity components + v:pathMilestoneCode::string as path_milestone_code, + v:pathMilestoneDescription::string as path_milestone_description, + -- edfi extensions + v:_ext as v_ext + + from path_milestones +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__path_phases.sql b/models/staging/tpdm/base/base_tpdm__path_phases.sql new file mode 100644 index 0000000..dab6a3f --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__path_phases.sql @@ -0,0 +1,31 @@ +with path_phases as ( + {{ source_edfi3('path_phases') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathPhaseName::string as path_phase_name, + v:pathReference:educationOrganizationId::int as ed_org_id, + v:pathReference:pathName::string as path_name, + -- non-identity components + v:pathPhaseSequence::int as path_phase_sequence, + v:phasePathDescription::string as phase_path_description, + -- unflattened lists + v:pathMilestones as v_path_milestones, + -- references + v:pathReference as path_reference, + -- edfi extensions + v:_ext as v_ext + + from path_phases +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__paths.sql b/models/staging/tpdm/base/base_tpdm__paths.sql new file mode 100644 index 0000000..caccda2 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__paths.sql @@ -0,0 +1,30 @@ +with paths as ( + {{ source_edfi3('paths') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathName::string as path_name, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:graduationPlanReference:educationOrganizationId::int as graduation_plan_ed_org_id, + {{ extract_descriptor('v:graduationPlanReference:graduationPlanTypeDescriptor::string') }} as graduation_plan_type, + v:graduationPlanReference:graduationSchoolYear::int as graduation_school_year, + -- references + v:educationOrganizationReference as education_organization_reference, + v:graduationPlanReference as graduation_plan_reference, + -- edfi extensions + v:_ext as v_ext + + from paths +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql new file mode 100644 index 0000000..d62669d --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql @@ -0,0 +1,32 @@ +with professional_development_event_attendances as ( + {{ source_edfi3('professional_development_event_attendances') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:attendanceDate::date as attendance_date, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, + v:professionalDevelopmentEventReference:namespace::string as professional_development_event_namespace, + v:professionalDevelopmentEventReference:professionalDevelopmentTitle::string as professional_development_title, + -- non-identity components + v:attendanceEventReason::string as attendance_event_reason, + -- descriptors + {{ extract_descriptor('v:attendanceEventCategoryDescriptor::string') }} as attendance_event_category, + -- references + v:personReference as person_reference, + v:professionalDevelopmentEventReference as professional_development_event_reference, + -- edfi extensions + v:_ext as v_ext + from professional_development_event_attendances +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_events.sql b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql new file mode 100644 index 0000000..5d87f3a --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql @@ -0,0 +1,30 @@ +with professional_development_events as ( + {{ source_edfi3('professional_development_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:professionalDevelopmentTitle::string as professional_development_title, + v:namespace::string as namespace, + -- non-identity components + v:multipleSession::boolean as is_multiple_session, + v:professionalDevelopmentReason::string as professional_development_reason, + v:required::boolean as is_required, + v:totalHours::integer as total_hours, -- should this be float or number type instead? + -- descriptors + {{ extract_descriptor('v:professionalDevelopmentOfferedByDescriptor::string') }} as offered_by, + -- edfi extensions + v:_ext as v_ext + + from professional_development_events +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql b/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql new file mode 100644 index 0000000..70ee31d --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql @@ -0,0 +1,50 @@ +with quantitative_measure_scores as ( + {{ source_edfi3('quantitative_measure_scores') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:quantitativeMeasureReference:quantitativeMeasureIdentifier::string as quantitative_measure_identifier, + v:quantitativeMeasureReference:educationOrganizationId::int as quant_msr_ed_org_id, + v:quantitativeMeasureReference:evaluationElementTitle::string as quant_msr_evaluation_element_title, + v:quantitativeMeasureReference:evaluationObjectiveTitle::string as quant_msr_evaluation_objective_title, + {{ extract_descriptor('v:quantitativeMeasureReference:evaluationPeriodDescriptor::string') }} as quant_msr_evaluation_period, + v:quantitativeMeasureReference:evaluationTitle::string as quant_msr_evaluation_title, + v:quantitativeMeasureReference:performanceEvaluationTitle::string as quant_msr_performance_evaluation_title, + {{ extract_descriptor('v:quantitativeMeasureReference:performanceEvaluationTypeDescriptor::string') }} as quant_msr_performance_evaluation_type, + v:quantitativeMeasureReference:schoolYear::int as quant_msr_school_year, + {{ extract_descriptor('v:quantitativeMeasureReference:termDescriptor::string') }} as quant_msr_academic_term, + + v:evaluationElementRatingReference:evaluationDate::date as eval_elem_rating_evaluation_date, + v:evaluationElementRatingReference:personId::string as eval_elem_rating_person_id, + v:evaluationElementRatingReference:educationOrganizationId::int as eval_elem_rating_ed_org_id, + v:evaluationElementRatingReference:evaluationElementTitle::string as eval_elem_rating_evaluation_element_title, + v:evaluationElementRatingReference:evaluationObjectiveTitle::string as eval_elem_rating_evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementRatingReference:evaluationPeriodDescriptor::string') }} as eval_elem_rating_evaluation_period, + v:evaluationElementRatingReference:evaluationTitle::string as eval_elem_rating_evaluation_title, + v:evaluationElementRatingReference:performanceEvaluationTitle::string as eval_elem_rating_performance_evaluation_title, + {{ extract_descriptor('v:evaluationElementRatingReference:performanceEvaluationTypeDescriptor::string') }} as eval_elem_rating_evaluation_type, + v:evaluationElementRatingReference:schoolYear::int as eval_elem_rating_school_year, + {{ extract_descriptor('v:evaluationElementRatingReference:termDescriptor::string') }} as eval_elem_rating_academic_term, + {{ extract_descriptor('v:evaluationElementRatingReference:sourceSystemDescriptor::string') }} as eval_elem_rating_source_system, + -- non-identity components + v:scoreValue::string as score_value, + v:standardError::string as standard_error, + -- references + v:quantitativeMeasureReference as quantitative_measure_reference, + v:evaluationElementRatingReference as evaluation_element_rating_reference, + -- edfi extensions + v:_ext as v_ext + + from quantitative_measure_scores +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql b/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql new file mode 100644 index 0000000..14215e6 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql @@ -0,0 +1,36 @@ +with quantitative_measures as ( + {{ source_edfi3('quantitative_measures') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:evaluationElementReference:evaluationTitle::string as evaluation_title, + v:evaluationElementReference:evaluationElementTitle::string as evaluation_element_title, + v:evaluationElementReference:evaluationObjectiveTitle::string as evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementReference:evaluationPeriodDescriptor::string') }} as evaluation_period, + v:evaluationElementReference:performanceEvaluationTitle::string as performance_evaluation_title, + v:evaluationElementReference:performanceEvaluationTypeDescriptor::string as performance_evaluation_type, + v:evaluationElementReference:educationOrganizationId::int as ed_org_id, + v:evaluationElementReference:schoolYear::int as school_year, + {{ extract_descriptor('v:evaluationElementReference:termDescriptor::string') }} as academic_term, + v:quantitativeMeasureIdentifier::string as quantitative_measure_identifier, + -- descriptors + {{ extract_descriptor('v:quantitativeMeasureTypeDescriptor::string') }} as quantitative_measure_type, + {{ extract_descriptor('v:quantitativeMeasureDatatypeDescriptor::string') }} as quantitative_measure_datatype, + -- references + v:evaluationElementReference as evaluation_element_reference, + -- edfi extensions + v:_ext as v_ext + + from quantitative_measures +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql b/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql new file mode 100644 index 0000000..93420f2 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql @@ -0,0 +1,63 @@ +with recruitment_event_attendances as ( + {{ source_edfi3('recruitment_event_attendances') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:recruitmentEventAttendeeIdentifier::string as attendee_identifier, + v:recruitmentEventReference:eventTitle::string as event_title, + v:recruitmentEventReference:eventDate::date as event_date, + v:recruitmentEventReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:currentPosition:positionTitle::string as current_position_title, + {{ extract_descriptor('v:currentPosition:academicSubjectDescriptor::string') }} as current_position_academic_subject, + v:currentPosition:nameOfInstitution::string as current_position_institution_name, + v:currentPosition:location::string as current_position_location, + v:firstName::string as first_name, + v:lastSurname::string as last_surname, + v:maidenName::string as maiden_name, + v:middleName::string as middle_name, + v:generationCodeSuffix::string as generation_code_suffix, + v:personalTitlePrefix::string as personal_title_prefix, + v:electronicMailAddress::string as email_address, + v:socialMediaNetworkName::string as social_media_network_name, + v:socialMediaUserName::string as social_media_username, + v:met::boolean as was_met, -- indicates if person was met by a representative of the ed org, is there a better naming? + v:preScreeningRating::integer as pre_screening_rating, + v:applied::boolean as has_applied, -- indicates whether the prospect applied for a position. There might be a better name for this. + v:recruitmentEventAttendeeQualifications:eligible::boolean as is_eligible, + v:recruitmentEventAttendeeQualifications:capacityToServe::boolean as capacity_to_serve, + v:recruitmentEventAttendeeQualifications:yearsOfServiceCurrentPlacement::float as years_of_service_current_placement, + v:recruitmentEventAttendeeQualifications:yearsOfServiceTotal::float as years_of_service_total, + v:referral::boolean as is_referral, + v:referredBy::string as referred_by, + v:notes::string as additional_notes, + v:hispanicLatinoEthnicity::boolean as has_hispanic_latino_ethnicity, + -- descriptors + {{ extract_descriptor('v:genderDescriptor::string') }} as gender, + {{ extract_descriptor('v:sexDescriptor::string') }} as sex, + {{ extract_descriptor('v:recruitmentEventAttendeeTypeDescriptor::string') }} as recruitment_event_attendee_type, + -- unflattened lists + v:currentPosition:gradeLevels as current_position_grade_levels, -- is this the correct way of naming a list that is nested within a composit part? + v:disabilities as v_disabilities, + v:personalIdentificationDocuments as v_personal_identification_documents, + v:races as v_races, + v:telephones as v_telephones, + v:touchpoints as v_touchpoints, + -- references + v:recruitmentEventReference as recruitment_event_reference, + -- edfi extensions + v:_ext as v_ext + + from recruitment_event_attendances +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__recruitment_events.sql b/models/staging/tpdm/base/base_tpdm__recruitment_events.sql new file mode 100644 index 0000000..49458d6 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__recruitment_events.sql @@ -0,0 +1,31 @@ +with recruitment_events as ( + {{ source_edfi3('recruitment_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:eventDate::date as event_date, + v:eventTitle::string as event_title, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:eventDescription::string as event_description, + v:eventLocation::string as event_location, + -- descriptors + {{ extract_descriptor('v:recruitmentEventTypeDescriptor::string') }} as recruitment_event_type, + -- references + v:educationOrganizationReference as education_organization_reference, + -- edfi extensions + v:_ext as v_ext + + from recruitment_events +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql b/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql new file mode 100644 index 0000000..af389cc --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql @@ -0,0 +1,32 @@ +with staff_educator_preparation_program_associations as ( + {{ source_edfi3('staff_educator_preparation_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:staffReference:staffUniqueId::string as staff_id, + v:educatorPreparationProgramReference:educationOrganizationId::int as ed_org_id, + v:educatorPreparationProgramReference:programName::string as program_name, + {{ extract_descriptor('v:educatorPreparationProgramReference:programTypeDescriptor') }} as program_type, + -- non-identity components + v:beginDate::date as begin_date, + v:educatorPreparationProgramReference:link:rel::string as ed_org_type, -- this was declareed in base_tpdm__candidate_educator_preparation_program_associations, is it correct? + v:endDate::date as end_date, + v:completer::boolean as is_completer, + -- references + v:staffReference as staff_reference, + v:educatorPreparationProgramReference as educator_preparation_program_reference, + -- edfi extensions + v:_ext as v_ext + from staff_educator_preparation_program_associations +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql b/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql new file mode 100644 index 0000000..deb24f1 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql @@ -0,0 +1,37 @@ +with student_path_milestone_statuses as ( + {{ source_edfi3('student_path_milestone_statuses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathMilestoneReference:pathMilestoneName::string as path_milestone_name, + {{ extract_descriptor('v:pathMilestoneReference:pathMilestoneTypeDescriptor::string') }} as path_milestone_type, + v:studentPathReference:educationOrganizationId::int as ed_org_id, + v:studentPathReference:pathName::string as path_name, + v:studentPathReference:studentUniqueId::string as student_unique_id, + -- non-identity components + v:completionIndicator::boolean as has_completed, + v:event:description::string as status_description, + v:event:pathMilestoneStatusDate::string as status_date, + -- descriptors + {{ extract_descriptor('v:event:pathMilestoneStatusDescriptor::string') }} as status_descriptor, + -- references + v:pathMilestoneReference as path_milestone_reference, + v:studentPathReference as student_path_reference, + v:pathPhaseReference as path_phase_reference, + + -- edfi extensions + v:_ext as v_ext + + from student_path_milestone_statuses +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql b/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql new file mode 100644 index 0000000..1dba9b6 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql @@ -0,0 +1,35 @@ +with student_path_phase_statuses as ( + {{ source_edfi3('student_path_phase_statuses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathPhaseReference:pathName::string as path_phase_path_name, + v:pathPhaseReference:pathPhaseName::string as path_phase_name, + v:pathPhaseReference:educationOrganizationId::int as path_phase_ed_org_id, + v:studentPathReference:pathName::string as student_path_path_name, + v:studentPathReference:studentUniqueId::string as student+unique_id, + v:studentPathReference:educationOrganizationId::int as student_path_ed_org_id, + -- non-identity components + v:completionIndicator::boolean as has_completed, + -- unflattened lists + v:events as v_events, + v:periods as v_periods, + -- references + v:pathPhaseReference as path_phase_reference, + v:studentPathReference as student_path_reference, + -- edfi extensions + v:_ext as v_ext + + from student_path_phase_statuses +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__student_paths.sql b/models/staging/tpdm/base/base_tpdm__student_paths.sql new file mode 100644 index 0000000..afc8c85 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_paths.sql @@ -0,0 +1,29 @@ +with student_paths as ( + {{ source_edfi3('student_paths') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathReference:pathName::string as path_name, + v:studentReference:studentUniqueId::string as student_unique_id, + v:pathReference:educationOrganizationId::int as ed_org_id, + -- unflattened lists + v:periods as v_periods, + -- references + v:pathReference as path_reference, + v:studentReference as student_reference, + -- edfi extensions + v:_ext as v_ext + + from student_paths +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql new file mode 100644 index 0000000..9afbd19 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql @@ -0,0 +1,40 @@ +with survey_section_aggregate_responses as ( + {{ source_edfi3('survey_section_aggregate_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:evaluationElementRatingReference:educationOrganizationId::string as ed_org_id, + v:evaluationElementRatingReference:evaluationDate::string as evaluation_date, + v:evaluationElementRatingReference:evaluationElementTitle::string as evaluation_element_title, + {{ extract_descriptor('v:evaluationElementRatingReference:sourceSystemDescriptor::string') }} as evaluation_element_rating_source_system, + v:evaluationElementRatingReference:evaluationObjectiveTitle::string as evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementRatingReference:evaluationPeriodDescriptor::string') }} as evaluation_period, + v:evaluationElementRatingReference:evaluationTitle::string as evaluation_title, + v:evaluationElementRatingReference:performanceEvaluationTitle::string as performance_evaluation_title, + {{ extract_descriptor('v:evaluationElementRatingReference:performanceEvaluationTypeDescriptor::string') }} as performance_evaluation_type, + v:evaluationElementRatingReference:personId::string as person_id, + v:evaluationElementRatingReference:schoolYear::string as school_year, + {{ extract_descriptor('v:evaluationElementRatingReference:termDescriptor::string') }} as academic_term, + v:surveySectionReference:surveyIdentifier::string as survey_identifier, + v:surveySectionReference:namespace::string as survey_section_namespace, + v:surveySectionReference:surveySectionTitle::string as survey_section_title, + -- non-identity components + v:scoreValue::float as score_value, + -- references + v:evaluationElementRatingReference as evaluation_element_rating_reference, + v:surveySectionReference as survey_section_reference, + -- edfi extensions + v:_ext as v_ext + from survey_section_aggregate_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql new file mode 100644 index 0000000..f5563e8 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -0,0 +1,29 @@ +with survey_section_response_person_target_associations as ( + {{ source_edfi3('survey_section_response_person_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:surveySectionResponseReference:namespace::string as namespace, + v:surveySectionResponseReference:surveyIdentifier::string as survey_id, + v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, + v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_identifier, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, + -- references + v:surveySectionResponseReference as survey_section_response_reference, + v:personReference as person_reference, + -- edfi extensions + v:_ext as v_ext + from survey_section_response_person_target_associations +) +select * from renamed diff --git a/models/staging/tpdm/stage/_tpdm__stage.yml b/models/staging/tpdm/stage/_tpdm__stage.yml index 2e19a4a..dff4c65 100644 --- a/models/staging/tpdm/stage/_tpdm__stage.yml +++ b/models/staging/tpdm/stage/_tpdm__stage.yml @@ -90,6 +90,12 @@ referential_integrity_tests: field: k_evaluation_objective_rating tags: ['ref_integrity'] + - k_survey_section_response: &ref_k_survey_section_response + - relationships: + to: ref('stg_ef3__survey_section_responses') + field: k_survey_section_response + tags: ['ref_integrity'] + models: - name: stg_tpdm__candidate_educator_preparation_program_associations config: @@ -274,3 +280,15 @@ models: - name: k_educator_prep_program tests: *ref_k_educator_prep_program + + - name: stg_tpdm__survey_section_response_person_target_associations + config: + tags: ['tpdm'] + enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" + columns: + - name: k_person + tests: + *ref_k_person + - name: k_survey_section_response + tests: + *ref_k_survey_section_response diff --git a/models/staging/tpdm/stage/stg_tpdm__application_events.sql b/models/staging/tpdm/stage/stg_tpdm__application_events.sql new file mode 100644 index 0000000..15f4eff --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__application_events.sql @@ -0,0 +1,21 @@ +with base_app_events as ( + select * from {{ ref('base_tpdm__application_events')}} +), +keyed as ( + select + {{ gen_skey('k_application') }}, + base_app_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_app_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_application, sequence_number, event_date, application_event_type', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__credential_events.sql b/models/staging/tpdm/stage/stg_tpdm__credential_events.sql new file mode 100644 index 0000000..30b6557 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__credential_events.sql @@ -0,0 +1,21 @@ +with base_credential_events as ( + select * from {{ ref('base_tpdm__credential_events')}} +), +keyed as ( + select + {{ gen_skey('k_credential') }}, + base_credential_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_credential_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_credential, credential_event_date, credential_event_type, credential_identifier', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql new file mode 100644 index 0000000..b2ee0a9 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql @@ -0,0 +1,22 @@ +with base_student_section_associations as ( + select * from {{ ref('stg_tpdm__fieldwork_experience_section_associations') }} +), +keyed as ( + select + {{ gen_skey('k_fieldwork_experience') }}, + {{ gen_skey('k_section') }}, -- should this be k_course_section instead? + base_student_section_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_section_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_fieldwork_experience, k_section', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__goals.sql b/models/staging/tpdm/stage/stg_tpdm__goals.sql new file mode 100644 index 0000000..74c1b1d --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__goals.sql @@ -0,0 +1,34 @@ +with base_goals as ( + select * from {{ ref('base_tpdm__goals') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'assignment_date', + 'lower(goal_title)', + 'lower(person_id)', + 'lower(person_source_system)' + ] + ) }} as k_goal, + {{ gen_skey('k_person') }}, + {{ gen_skey('k_evaluation_element') }}, + {{ gen_skey('k_evaluation_objective') }}, + {{ gen_skey('k_goal', alt_ref='parent_goal_reference', alt_k_name='k_goal__parent') }}, + base_goals.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_goals +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_goal', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql b/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql new file mode 100644 index 0000000..1f7659a --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql @@ -0,0 +1,21 @@ +with open_staff_position_events as ( + select * from {{ ref('base_tpdm__open_staff_position_events')}} +), +keyed as ( + select + {{ gen_skey('k_open_staff_position') }}, + open_staff_position_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from open_staff_position_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_open_staff_position, event_date, open_staff_position_event_type', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql b/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql new file mode 100644 index 0000000..e9bb4d2 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql @@ -0,0 +1,28 @@ +with base_path_milestones as ( + select * from {{ ref('base_tpdm__path_milestones') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(path_milestone_name)', + 'lower(path_milestone_type)' + ] + ) }} as k_path_milestone, + base_path_milestones.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_path_milestones +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_milestone', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__path_phases.sql b/models/staging/tpdm/stage/stg_tpdm__path_phases.sql new file mode 100644 index 0000000..7e3c80a --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__path_phases.sql @@ -0,0 +1,30 @@ +with base_path_phases as ( + select * from {{ ref('base_tpdm__path_phases') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)', + 'lower(path_phase_name)' + ] + ) }} as k_path_phase, + {{ gen_skey('k_path') }}, + base_path_phases.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_path_phases +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_phase', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__paths.sql b/models/staging/tpdm/stage/stg_tpdm__paths.sql new file mode 100644 index 0000000..368671e --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__paths.sql @@ -0,0 +1,30 @@ +with base_paths as ( + select * from {{ ref('base_tpdm__paths') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)' + ] + ) }} as k_path, + {{ gen_skey('k_ed_org') }}, + {{ gen_skey('k_graduation_plan') }}, + base_paths.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_paths +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql b/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql new file mode 100644 index 0000000..e62c7a6 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql @@ -0,0 +1,22 @@ +with professional_dev_event_attendances as ( + select * from {{ ref('base_tpdm__professional_development_event_attendances')}} +), +keyed as ( + select + {{ gen_skey('k_person') }}, + {{ gen_skey('k_professional_development_event') }}, + professional_dev_event_attendances.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from professional_dev_event_attendances +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_person, k_professional_development_event, attendance_date', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql b/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql new file mode 100644 index 0000000..39f779b --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql @@ -0,0 +1,28 @@ +with professional_dev_events as ( + select * from {{ ref('base_tpdm__professional_development_events') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(namespace)', + 'lower(professional_development_title)' + ] + ) }} as k_professional_development_event, + professional_dev_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from professional_dev_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_professional_development_event', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql b/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql new file mode 100644 index 0000000..e005919 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql @@ -0,0 +1,22 @@ +with base_quantitative_measure_scores as ( + select * from {{ ref('base_tpdm__quantitative_measure_scores') }} +), +keyed as ( + select + {{ gen_skey('k_quantitative_measure') }}, + {{ gen_skey('k_evaluation_element_rating') }}, + base_quantitative_measure_scores.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_quantitative_measure_scores +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_evaluation_element_rating, k_quantitative_measure', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql b/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql new file mode 100644 index 0000000..fa464c8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql @@ -0,0 +1,37 @@ +with base_quantitative_measures as ( + select * from {{ ref('base_tpdm__quantitative_measures') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(evaluation_element_title)', + 'lower(evaluation_objective_title)', + 'lower(evaluation_period)', + 'lower(evaluation_title)', + 'lower(performance_evaluation_title)', + 'lower(performance_evaluation_type)', + 'lower(quantitative_measure_identifier)', + 'school_year', + 'lower(academic_term)' + ] + ) }} as k_quantitative_measure, + {{ gen_skey('k_evaluation_element') }}, + base_quantitative_measures.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_quantitative_measures +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_quantitative_measure', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql b/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql new file mode 100644 index 0000000..aefb359 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql @@ -0,0 +1,31 @@ +with base_recruitment_event_attendances as ( + select * from {{ ref('base_tpdm__recruitment_event_attendances') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(attendee_identifier)', + 'ed_org_id', + 'event_date', + 'lower(event_title)' + ] + ) }} as k_recruitment_event_attendance, + {{ gen_skey('k_recruitment_event') }}, + base_recruitment_event_attendances.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_recruitment_event_attendances +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_recruitment_event_attendance', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql new file mode 100644 index 0000000..705d47b --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql @@ -0,0 +1,30 @@ +with base_recruitment_events as ( + select * from {{ ref('base_tpdm__recruitment_events') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'event_date', + 'lower(event_title)', + ] + ) }} as k_recruitment_event, + {{ gen_skey('k_ed_org') }}, + base_recruitment_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_recruitment_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_recruitment_event', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__staff_educator_preparation_program_associations.sql b/models/staging/tpdm/stage/stg_tpdm__staff_educator_preparation_program_associations.sql new file mode 100644 index 0000000..8d4f448 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__staff_educator_preparation_program_associations.sql @@ -0,0 +1,22 @@ +with staff_educator_preparation_program_associations as ( + select * from {{ ref('base_tpdm__staff_educator_preparation_program_associations') }} +), +keyed as ( + select + {{ gen_skey('k_staff') }}, + {{ gen_skey('k_educator_prep_program') }}, + staff_educator_preparation_program_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from staff_educator_preparation_program_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_staff, k_educator_prep_program', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted diff --git a/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql b/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql new file mode 100644 index 0000000..844bfc5 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql @@ -0,0 +1,22 @@ +with base_student_path_milestone_statuses as ( + select * from {{ ref('base_tpdm__student_path_milestone_statuses') }} +), +keyed as ( + select + {{ gen_skey('k_student_path') }}, + {{ gen_skey('k_path_milestone') }}, + base_student_path_milestone_statuses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_path_milestone_statuses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_path, k_path_milestone', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql b/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql new file mode 100644 index 0000000..7306a9d --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql @@ -0,0 +1,22 @@ +with base_student_path_phase_statuses as ( + select * from {{ ref('base_tpdm__student_path_phase_statuses') }} +), +keyed as ( + select + {{ gen_skey('k_path_phase') }}, + {{ gen_skey('k_student_path') }}, + base_student_path_phase_statuses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_path_phase_statuses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_phase, k_student_path', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__student_paths.sql b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql new file mode 100644 index 0000000..2c9b472 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql @@ -0,0 +1,31 @@ +with base_student_paths as ( + select * from {{ ref('base_tpdm__student_paths') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)', + 'lower(student_unique_id)' + ] + ) }} as k_student_path, + {{ gen_skey('k_path') }}, + {{ gen_skey('k_student') }}, + base_student_paths.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_paths +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_path', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql new file mode 100644 index 0000000..afd5802 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql @@ -0,0 +1,21 @@ +with base_survey_section_aggregate_responses as ( + select * from {{ ref('base_tpdm__survey_section_aggregate_responses') }} +), +keyed as ( + select + {{ gen_skey('k_evaluation_element_rating') }}, + {{ gen_skey('k_survey_section') }}, + base_survey_section_aggregate_responses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_survey_section_aggregate_responses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section_response, k_survey_section', + order_by='last_modified_timestamp desc, pull_timestamp desc') + }} +) +select * from deduped +where not is_deleted diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql new file mode 100644 index 0000000..7015ce2 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql @@ -0,0 +1,21 @@ +with survey_section_response_person_target_associations as ( + select * from {{ ref('base_tpdm__survey_section_response_person_target_associations') }} +), +keyed as ( + select + {{ gen_skey('k_survey_section_response') }}, + {{ gen_skey('k_person') }}, + survey_section_response_person_target_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from survey_section_response_person_target_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section_response, k_person', + order_by='last_modified_timestamp desc, pull_timestamp desc') + }} +) +select * from deduped +where not is_deleted