58f8093689
- Updated all component headers and documentation
- Changed navbar and footer branding
- Updated homepage hero badge
- Modified page title in index.html
- Simplified footer text to 'Built with ❤️'
- Consistent V2 capitalization across all references
268 lines
9.4 KiB
JavaScript
268 lines
9.4 KiB
JavaScript
const content = `--[[
|
|
Remove jobs from the specific set.
|
|
Input:
|
|
KEYS[1] set key,
|
|
KEYS[2] events stream key
|
|
ARGV[1] jobKey prefix
|
|
ARGV[2] timestamp
|
|
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
|
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
|
]]
|
|
local rcall = redis.call
|
|
local rangeStart = 0
|
|
local rangeEnd = -1
|
|
local limit = tonumber(ARGV[3])
|
|
-- If we're only deleting _n_ items, avoid retrieving all items
|
|
-- for faster performance
|
|
--
|
|
-- Start from the tail of the list, since that's where oldest elements
|
|
-- are generally added for FIFO lists
|
|
if limit > 0 then
|
|
rangeStart = -1 - limit + 1
|
|
rangeEnd = -1
|
|
end
|
|
-- Includes
|
|
--[[
|
|
Function to clean job list.
|
|
Returns jobIds and deleted count number.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to get the latest saved timestamp.
|
|
]]
|
|
local function getTimestamp(jobKey, attributes)
|
|
if #attributes == 1 then
|
|
return rcall("HGET", jobKey, attributes[1])
|
|
end
|
|
local jobTs
|
|
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
|
if (ts) then
|
|
jobTs = ts
|
|
break
|
|
end
|
|
end
|
|
return jobTs
|
|
end
|
|
--[[
|
|
Function to remove job.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Check if this job has a parent. If so we will just remove it from
|
|
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
|
which requires code from "moveToFinished"
|
|
]]
|
|
--[[
|
|
Functions to destructure job key.
|
|
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
|
]]
|
|
local getJobIdFromKey = function (jobKey)
|
|
return string.match(jobKey, ".*:(.*)")
|
|
end
|
|
local getJobKeyPrefix = function (jobKey, jobId)
|
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
|
end
|
|
--[[
|
|
Function to check for the meta.paused key to decide if we are paused or not
|
|
(since an empty list and !EXISTS are not really the same).
|
|
]]
|
|
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
|
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
|
return waitKey, false
|
|
else
|
|
return pausedKey, true
|
|
end
|
|
end
|
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
|
rcall("RPUSH", parentTarget, parentId)
|
|
if emitEvent then
|
|
local parentEventStream = parentPrefix .. "events"
|
|
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
|
end
|
|
end
|
|
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
if parentKey then
|
|
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
if result > 0 then
|
|
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
if pendingDependencies == 0 then
|
|
local parentId = getJobIdFromKey(parentKey)
|
|
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
|
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
if numRemovedElements == 1 then
|
|
if hard then
|
|
if parentPrefix == baseKey then
|
|
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
|
rcall("DEL", parentKey, parentKey .. ':logs',
|
|
parentKey .. ':dependencies', parentKey .. ':processed')
|
|
else
|
|
moveParentToWait(parentPrefix, parentId)
|
|
end
|
|
else
|
|
moveParentToWait(parentPrefix, parentId, true)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
else
|
|
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
|
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
if result > 0 then
|
|
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
if pendingDependencies == 0 then
|
|
local parentId = getJobIdFromKey(missedParentKey)
|
|
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
if numRemovedElements == 1 then
|
|
if hard then
|
|
if parentPrefix == baseKey then
|
|
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
|
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
|
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
|
else
|
|
moveParentToWait(parentPrefix, parentId)
|
|
end
|
|
else
|
|
moveParentToWait(parentPrefix, parentId, true)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
local function removeJob(jobId, hard, baseKey)
|
|
local jobKey = baseKey .. jobId
|
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
rcall("DEL", jobKey, jobKey .. ':logs',
|
|
jobKey .. ':dependencies', jobKey .. ':processed')
|
|
end
|
|
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
|
|
timestamp, isWaiting)
|
|
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
|
local deleted = {}
|
|
local deletedCount = 0
|
|
local jobTS
|
|
local deletionMarker = ''
|
|
local jobIdsLen = #jobs
|
|
for i, job in ipairs(jobs) do
|
|
if limit > 0 and deletedCount >= limit then
|
|
break
|
|
end
|
|
local jobKey = jobKeyPrefix .. job
|
|
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
|
|
-- the job is first tried
|
|
-- * timestamp is the original job submission time
|
|
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
|
|
-- that have been active within the grace period:
|
|
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
|
if (not jobTS or jobTS <= timestamp) then
|
|
-- replace the entry with a deletion marker; the actual deletion will
|
|
-- occur at the end of the script
|
|
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
|
removeJob(job, true, jobKeyPrefix)
|
|
deletedCount = deletedCount + 1
|
|
table.insert(deleted, job)
|
|
end
|
|
end
|
|
end
|
|
rcall("LREM", listKey, 0, deletionMarker)
|
|
return {deleted, deletedCount}
|
|
end
|
|
--[[
|
|
Function to clean job set.
|
|
Returns jobIds and deleted count number.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to loop in batches.
|
|
Just a bit of warning, some commands as ZREM
|
|
could receive a maximum of 7000 parameters per call.
|
|
]]
|
|
local function batches(n, batchSize)
|
|
local i = 0
|
|
return function()
|
|
local from = i * batchSize + 1
|
|
i = i + 1
|
|
if (from <= n) then
|
|
local to = math.min(from + batchSize - 1, n)
|
|
return from, to
|
|
end
|
|
end
|
|
end
|
|
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
|
|
-- of items in a sorted set only run a single iteration. If we simply used
|
|
-- ZRANGE, we may take a long time traversing through jobs that are within the
|
|
-- grace period.
|
|
local function getJobsInZset(zsetKey, rangeEnd, limit)
|
|
if limit > 0 then
|
|
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
|
|
else
|
|
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
|
|
end
|
|
end
|
|
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
|
|
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
|
local deleted = {}
|
|
local deletedCount = 0
|
|
local jobTS
|
|
for i, job in ipairs(jobs) do
|
|
if limit > 0 and deletedCount >= limit then
|
|
break
|
|
end
|
|
local jobKey = jobKeyPrefix .. job
|
|
if isFinished then
|
|
removeJob(job, true, jobKeyPrefix)
|
|
deletedCount = deletedCount + 1
|
|
table.insert(deleted, job)
|
|
else
|
|
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
jobTS = getTimestamp(jobKey, attributes)
|
|
if (not jobTS or jobTS <= timestamp) then
|
|
removeJob(job, true, jobKeyPrefix)
|
|
deletedCount = deletedCount + 1
|
|
table.insert(deleted, job)
|
|
end
|
|
end
|
|
end
|
|
if(#deleted > 0) then
|
|
for from, to in batches(#deleted, 7000) do
|
|
rcall("ZREM", setKey, unpack(deleted, from, to))
|
|
end
|
|
end
|
|
return {deleted, deletedCount}
|
|
end
|
|
local result
|
|
if ARGV[4] == "active" then
|
|
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
|
elseif ARGV[4] == "delayed" then
|
|
rangeEnd = "+inf"
|
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
|
{"processedOn", "timestamp"}, false)
|
|
elseif ARGV[4] == "prioritized" then
|
|
rangeEnd = "+inf"
|
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
|
{"timestamp"}, false)
|
|
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
|
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
|
else
|
|
rangeEnd = ARGV[2]
|
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
|
{"finishedOn"}, true)
|
|
end
|
|
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
|
return result[1]
|
|
`;
|
|
export const cleanJobsInSet = {
|
|
name: 'cleanJobsInSet',
|
|
content,
|
|
keys: 2,
|
|
};
|
|
//# sourceMappingURL=cleanJobsInSet-2.js.map
|