Skip to content

Commit cd6ec2d

Browse files
Que migration v8: que_jobs_ext view; add first_run_at column to que_jobs
FIX bulk_insert_jobs and insert_jobs don't pass redundant arg for first_run_at (use run_at) cleanup. specs. documentation migrations and specs support backwards compatibility PR change requests. remove unnecessary db_version check in introspection spec. Add view and job table enhancements for improved observability
1 parent a0e652c commit cd6ec2d

File tree

11 files changed

+312
-12
lines changed

11 files changed

+312
-12
lines changed

docs/README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
- [Enqueueing jobs in bulk](#enqueueing-jobs-in-bulk)
5757
- [Expired jobs](#expired-jobs)
5858
- [Finished jobs](#finished-jobs)
59+
- [Que Jobs Ext view](#que-jobs-ext-view)
5960

6061
<!-- /MarkdownTOC -->
6162

@@ -879,3 +880,17 @@ SET LOCAL que.skip_notify TO true;
879880
DELETE FROM que_jobs WHERE finished_at < (select now() - interval '7 days');
880881
COMMIT;
881882
```
883+
884+
## Que Jobs Ext view
885+
886+
This view extends the functionality of the que job management system by providing an enriched view of que jobs. It combines data from the 'que_jobs' table and the 'pg_locks' table to present a comprehensive overview of que jobs, including their status, associated information, and locking details.
887+
888+
This view is designed to facilitate the monitoring and management of que jobs, allowing you to track job statuses, locking details, and job-related information.
889+
890+
Additional Columns:
891+
892+
- lock_id: Unique identifier for the lock associated with the job.
893+
- que_locker_pid: Process ID (PID) of the que job locker.
894+
- sub_class: The job class extracted from the job arguments.
895+
- updated_at: The most recent timestamp among 'run_at,' 'expired_at,' and 'finished_at.'
896+
- status: The status of the job, which can be 'running,' 'completed,' 'failed,' 'errored,' 'queued,' or 'scheduled.'

lib/que/job.rb

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ class Job
1212
SQL[:insert_job] =
1313
%{
1414
INSERT INTO public.que_jobs
15-
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
15+
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version, first_run_at)
1616
VALUES
1717
(
1818
coalesce($1, 'default')::text,
@@ -22,7 +22,8 @@ class Job
2222
coalesce($5, '[]')::jsonb,
2323
coalesce($6, '{}')::jsonb,
2424
coalesce($7, '{}')::jsonb,
25-
#{Que.job_schema_version}
25+
#{Que.job_schema_version},
26+
coalesce($3, now())::timestamptz
2627
)
2728
RETURNING *
2829
}
@@ -33,7 +34,7 @@ class Job
3334
SELECT * from json_to_recordset(coalesce($5, '[{args:{},kwargs:{}}]')::json) as x(args jsonb, kwargs jsonb)
3435
)
3536
INSERT INTO public.que_jobs
36-
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
37+
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version, first_run_at)
3738
SELECT
3839
coalesce($1, 'default')::text,
3940
coalesce($2, 100)::smallint,
@@ -42,7 +43,8 @@ class Job
4243
args_and_kwargs.args,
4344
args_and_kwargs.kwargs,
4445
coalesce($6, '{}')::jsonb,
45-
#{Que.job_schema_version}
46+
#{Que.job_schema_version},
47+
coalesce($3, now())::timestamptz
4648
FROM args_and_kwargs
4749
RETURNING *
4850
}
@@ -75,7 +77,8 @@ class << self
7577
:maximum_retry_count,
7678
:queue,
7779
:priority,
78-
:run_at
80+
:run_at,
81+
:first_run_at
7982

8083
def enqueue(*args)
8184
args, kwargs = Que.split_out_ruby2_keywords(args)

lib/que/migrations.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ module Que
44
module Migrations
55
# In order to ship a schema change, add the relevant up and down sql files
66
# to the migrations directory, and bump the version here.
7-
CURRENT_VERSION = 7
7+
CURRENT_VERSION = 8
88

99
class << self
1010
def migrate!(version:)

lib/que/migrations/8/down.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
DROP VIEW IF EXISTS public.que_jobs_ext;
2+
3+
ALTER TABLE que_jobs
4+
DROP COLUMN first_run_at;

lib/que/migrations/8/up.sql

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
-- Add 'first_run_at' column to 'que_jobs'. The 'first_run_at' column will store the timestamp with time zone (timestamptz)
2+
-- of the initial scheduled execution time for que jobs. The column default is set to now, but realistically this will always
3+
-- be defaulted to the initial run_at value. This enhancement helps re-trace the execution of the job when reviewing failures.
4+
5+
ALTER TABLE que_jobs
6+
ADD COLUMN first_run_at timestamptz NOT NULL DEFAULT now();
7+
8+
9+
-- This view extends the functionality of the que job management system by providing an enriched view of que jobs.
10+
-- It combines data from the 'que_jobs' table and the 'pg_locks' table to present a comprehensive overview of que jobs,
11+
-- including their status, associated information, and locking details. The view is designed to facilitate the monitoring
12+
-- and management of que jobs, allowing you to track job statuses, locking details, and job-related information.
13+
14+
-- Columns:
15+
-- - lock_id: Unique identifier for the lock associated with the job.
16+
-- - que_locker_pid: Process ID (PID) of the que job locker.
17+
-- - sub_class: The job class extracted from the job arguments.
18+
-- - updated_at: The most recent timestamp among 'run_at,' 'expired_at,' and 'finished_at.'
19+
-- - status: The status of the job, which can be 'running,' 'completed,' 'failed,' 'errored,' 'queued,' or 'scheduled.'
20+
21+
CREATE OR REPLACE VIEW public.que_jobs_ext
22+
AS
23+
SELECT
24+
locks.id AS lock_id,
25+
locks.pid as que_locker_pid,
26+
(que_jobs.args -> 0) ->> 'job_class'::text AS sub_class,
27+
greatest(run_at, expired_at, finished_at) as updated_at,
28+
29+
case
30+
when locks.id is not null then 'running'
31+
when finished_at is not null then 'completed'
32+
when expired_at is not null then 'failed'
33+
when error_count > 0 then 'errored'
34+
when run_at < now() then 'queued'
35+
else 'scheduled'
36+
end as status,
37+
38+
-- que_jobs.*:
39+
que_jobs.id,
40+
que_jobs.priority,
41+
que_jobs.run_at,
42+
que_jobs.first_run_at,
43+
que_jobs.job_class,
44+
que_jobs.error_count,
45+
que_jobs.last_error_message,
46+
que_jobs.queue,
47+
que_jobs.last_error_backtrace,
48+
que_jobs.finished_at,
49+
que_jobs.expired_at,
50+
que_jobs.args,
51+
que_jobs.data,
52+
que_jobs.kwargs,
53+
que_jobs.job_schema_version
54+
55+
FROM que_jobs
56+
LEFT JOIN (
57+
SELECT
58+
(classid::bigint << 32) + objid::bigint AS id
59+
, pid
60+
FROM pg_locks
61+
WHERE pg_locks.locktype = 'advisory'::text) locks USING (id);

lib/que/utils/queue_management.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def clear!
1212
# Very old migrations may use Que.create! and Que.drop!, which just
1313
# created and dropped the initial version of the jobs table.
1414
def create!; migrate!(version: 1); end
15-
def drop!; migrate!(version: 0); end
15+
def drop!; migrate!(version: 0); end
1616
end
1717
end
1818
end

spec/que/job.bulk_enqueue_spec.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def assert_enqueue(
4444
assert_equal expected_job_class.to_s, job[:job_class]
4545
assert_equal expected_args[i], job[:args]
4646
assert_equal expected_kwargs[i], job[:kwargs]
47+
assert_equal job[:run_at], job[:first_run_at]
4748
end
4849

4950
jobs_dataset.delete

spec/que/job.enqueue_spec.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ def assert_enqueue(
3939
assert_equal expected_job_class.to_s, job[:job_class]
4040
assert_equal expected_args, job[:args]
4141
assert_equal expected_job_schema_version, job[:job_schema_version]
42+
assert_equal job[:first_run_at], job[:run_at]
4243

4344
jobs_dataset.delete
4445
end

spec/que/utils/introspection_spec.rb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,10 @@
6565
locker.stop!
6666

6767
state = states.first
68-
assert_equal \
69-
%i(priority run_at id job_class error_count last_error_message queue
70-
last_error_backtrace finished_at expired_at args data job_schema_version kwargs ruby_hostname ruby_pid),
71-
state.keys
68+
expected_keys = %i(priority run_at id job_class error_count last_error_message queue
69+
last_error_backtrace finished_at expired_at args data job_schema_version kwargs ruby_hostname ruby_pid first_run_at)
70+
71+
assert_equal expected_keys.sort, state.keys.sort
7272

7373
assert_equal 2, state[:priority]
7474
assert_in_delta state[:run_at], Time.now, QueSpec::TIME_SKEW
Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
# frozen_string_literal: true
2+
3+
require 'spec_helper'
4+
5+
describe "Que Jobs Ext View", skip: true do
6+
7+
class TestJob < Que::Job
8+
include Que::JobMethods
9+
10+
def default_resolve_action
11+
# prevents default deletion of complete jobs for testing purposes
12+
finish
13+
end
14+
end
15+
16+
class TestFailedJob < TestJob
17+
def run
18+
raise Que::Error, 'Test Error'
19+
end
20+
end
21+
22+
describe 'job.enqueue' do
23+
it "should mirror enqueued job" do
24+
assert_equal 0, jobs_dataset.count
25+
assert_equal 0, jobs_ext_dataset.count
26+
27+
TestJob.enqueue(
28+
1,
29+
'two',
30+
string: "string",
31+
integer: 5,
32+
array: [1, "two", {three: 3}],
33+
hash: {one: 1, two: 'two', three: [3]},
34+
job_options: {
35+
priority: 4,
36+
queue: 'special_queue_name',
37+
run_at: Time.now
38+
}
39+
)
40+
41+
assert_equal 1, jobs_dataset.count
42+
assert_equal 1, jobs_ext_dataset.count
43+
44+
job = jobs_dataset.first
45+
ext_job = jobs_ext_dataset.first
46+
assert_equal ext_job[:queue], job[:queue]
47+
assert_equal ext_job[:priority], job[:priority]
48+
assert_equal ext_job[:run_at], job[:run_at]
49+
assert_equal ext_job[:first_run_at], job[:first_run_at]
50+
assert_equal ext_job[:job_class], job[:job_class]
51+
assert_equal ext_job[:args], job[:args]
52+
assert_equal ext_job[:job_schema_version], job[:job_schema_version]
53+
54+
jobs_dataset.delete
55+
56+
assert_equal 0, jobs_dataset.count
57+
assert_equal 0, jobs_ext_dataset.count
58+
end
59+
60+
it "should include additional lock data" do
61+
locker_settings.clear
62+
locker_settings[:listen] = false
63+
locker_settings[:poll_interval] = 0.02
64+
locker
65+
66+
TestJob.enqueue
67+
68+
sleep_until { locked_ids.count.positive? && locked_ids.first == jobs_ext_dataset.first[:lock_id] }
69+
70+
locker.stop!
71+
72+
jobs_dataset.delete
73+
end
74+
75+
it "should add additional updated_at" do
76+
TestJob.enqueue
77+
78+
ext_job = jobs_ext_dataset.first
79+
80+
assert_equal ext_job[:run_at], ext_job[:updated_at]
81+
82+
locker
83+
84+
sleep_until_equal(1) { finished_jobs_dataset.count }
85+
86+
locker.stop!
87+
88+
ext_job = jobs_ext_dataset.first
89+
90+
assert_equal ext_job[:finished_at], ext_job[:updated_at]
91+
92+
jobs_dataset.delete
93+
end
94+
95+
describe "should include additional status" do
96+
97+
let(:notified_errors) { [] }
98+
99+
it "should set status to scheduled when run_at is in the future" do
100+
TestJob.enqueue(job_options: { run_at: Time.now + 1 })
101+
102+
assert_equal jobs_ext_dataset.first[:status], 'scheduled'
103+
104+
jobs_dataset.delete
105+
end
106+
107+
it "should set status to queued when run_at is in the past and the job is not currently running, completed, failed or errored" do
108+
TestJob.enqueue(job_options: { run_at: Time.now - 1 })
109+
110+
assert_equal jobs_ext_dataset.first[:status], 'queued'
111+
112+
jobs_dataset.delete
113+
end
114+
115+
it "should set status to running when the job has a lock associated with it" do
116+
locker_settings.clear
117+
locker_settings[:listen] = false
118+
locker_settings[:poll_interval] = 0.02
119+
locker
120+
121+
TestJob.enqueue
122+
123+
sleep_until { locked_ids.count.positive? && locked_ids.first == jobs_ext_dataset.first[:lock_id] && jobs_ext_dataset.first[:status] == 'running' }
124+
125+
locker.stop!
126+
127+
jobs_dataset.delete
128+
end
129+
130+
it "should set status to complete when finished_at is present" do
131+
TestJob.enqueue
132+
133+
locker
134+
135+
sleep_until_equal(1) { DB[:que_lockers].count }
136+
137+
sleep_until { finished_jobs_dataset.count.positive? }
138+
139+
locker.stop!
140+
141+
assert_equal jobs_ext_dataset.first[:status], 'completed'
142+
143+
jobs_dataset.delete
144+
end
145+
146+
it "should set status to errored when error_count is positive and expired_at is not present" do
147+
Que.error_notifier = proc { |e| notified_errors << e }
148+
149+
TestFailedJob.class_eval do
150+
self.maximum_retry_count = 100 # prevent from entering failed state on first error
151+
end
152+
153+
locker
154+
155+
sleep_until_equal(1) { DB[:que_lockers].count }
156+
157+
TestFailedJob.enqueue
158+
159+
sleep_until { errored_jobs_dataset.where(expired_at: nil).count.positive? }
160+
161+
locker.stop!
162+
163+
ext_job = jobs_ext_dataset.first
164+
165+
assert_equal ext_job[:status], 'errored'
166+
assert_equal notified_errors.count, 1
167+
assert_equal notified_errors.first.message, 'Test Error'
168+
169+
170+
jobs_dataset.delete
171+
end
172+
173+
it "should set status to failed when expired_at is present" do
174+
TestFailedJob.class_eval do
175+
self.maximum_retry_count = 0
176+
end
177+
178+
Que.error_notifier = proc { |e| notified_errors << e }
179+
180+
locker
181+
182+
sleep_until_equal(1) { DB[:que_lockers].count }
183+
184+
TestFailedJob.enqueue
185+
186+
sleep_until { expired_jobs_dataset.count.positive? }
187+
188+
locker.stop!
189+
190+
assert_equal jobs_ext_dataset.first[:status], 'failed'
191+
assert_equal notified_errors.count, 1
192+
assert_equal notified_errors.first.message, 'Test Error'
193+
194+
195+
jobs_dataset.delete
196+
end
197+
end
198+
end
199+
end

0 commit comments

Comments
 (0)