From 47ce5866ee2bcaf4a36211c95657287dafd36bf8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 21:06:50 +0000 Subject: [PATCH 1/3] feat(core): recurring per-installation schedules behind a host-neutral port Integrations can declare recurring jobs (Definition.schedules, or a data-defined schedule trigger / polling sync). Core keeps one host schedule per installation in step through create, update, upgrade and delete. A developer can let users choose their own timing, with a floor. - recurrence.js: host-neutral { every } / five-field { cron, timezone }, validated by what an expression really does (min gap, never fires, dom+dow, time zones). - Scheduler port gains upsertRecurring; the EventBridge adapter translates to rate()/cron() (weekday names, `?`), updates or creates, survives a create race, and fills each firing's tick from context attributes; the mock adapter mirrors it. - Commands: upsertRecurringJob and reconcileRecurringJobs (writes only what changed; a refused write stays recorded as before and retries). - IntegrationBase.syncSchedules, called by the default lifecycle, records host state at config.runtime.schedules; invalid user choices keep the default and add a warning message. - Data-defined: compile emits schedule entries, validation and JSON Schemas accept configurable timing, the configure form offers it, versioning treats removing configurability as major and upgrades drop choices the new version disallows, ticks dedupe on their scheduled time. - devtools: Lambda may UpdateSchedule. 136 data-defined tests pass on real PostgreSQL and MongoDB; 15 mutations killed. Core's full suite fails the same 16 suites / 65 tests as before (pre-existing). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NGC9SnFuqdjX2xH2qeu8vw --- CLAUDE.md | 22 +- .../063-integration-definitions-as-data.md | 48 +++- .../commands/scheduler-commands.js | 95 +++++++- .../__tests__/recurring-schedules.test.js | 228 ++++++++++++++++++ .../eventbridge-scheduler-adapter.js | 70 +++++- .../core/infrastructure/scheduler/index.js | 9 +- .../scheduler/mock-scheduler-adapter.js | 33 ++- .../infrastructure/scheduler/recurrence.js | 169 +++++++++++++ .../scheduler/scheduler-service-interface.js | 36 ++- .../core/integrations/data-defined/README.md | 16 +- .../core/integrations/data-defined/compile.js | 10 +- .../integrations/data-defined/config-form.js | 27 +++ .../data-defined/data-defined-integration.js | 23 +- .../schema/installation.v0.1.schema.json | 86 ++++++- .../integration-definition.v0.1.schema.json | 66 ++++- .../data-defined/tests/schedules.test.js | 178 ++++++++++++++ .../integrations/data-defined/validate.js | 16 +- .../integrations/data-defined/versioning.js | 17 +- .../core/integrations/integration-base.js | 64 ++++- packages/core/integrations/schedules.js | 89 +++++++ .../tests/integration-base-schedules.test.js | 137 +++++++++++ .../domains/scheduler/scheduler-builder.js | 6 +- 22 files changed, 1408 insertions(+), 37 deletions(-) create mode 100644 packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js create mode 100644 packages/core/infrastructure/scheduler/recurrence.js create mode 100644 packages/core/integrations/data-defined/tests/schedules.test.js create mode 100644 packages/core/integrations/schedules.js create mode 100644 packages/core/integrations/tests/integration-base-schedules.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 5721194ad..74dd56661 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -917,7 +917,7 @@ When working on the Frigg Framework, always prioritize finding the **best soluti ### Scheduler Commands -The Frigg framework provides scheduler commands for scheduling one-time jobs using AWS EventBridge Scheduler: +The Frigg framework provides scheduler commands for one-time and recurring jobs using AWS EventBridge Scheduler: ```javascript const { createSchedulerCommands } = require('@friggframework/core'); @@ -942,8 +942,26 @@ await schedulerCommands.deleteJob('zoho-notif-renewal-abc123'); const status = await schedulerCommands.getJobStatus('zoho-notif-renewal-abc123'); ``` +**Recurring schedules, one per installation** (`Definition.schedules`): declare them on the integration and +`IntegrationBase` creates, retimes and deletes them in `onCreate` / `onUpdate` / `onDelete` (an override calls +`this.syncSchedules()` itself). Recurrence is host-neutral; each scheduler adapter translates it. + +```javascript +static Definition = { + name: 'housecallpro', + schedules: { + ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // user may set config.schedules.ONGOING_SYNC + DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed + }, +}; +``` + +The host's state is recorded at `config.runtime.schedules`. Each firing sends `{ event, data: { integrationId, +scheduleKey, tick: { id, scheduledFor } } }` to the integration's queue, so a handler can dedupe a redelivered tick. +For one job without the lifecycle, use `schedulerCommands.upsertRecurringJob({ jobId, recurrence, event, payload, queueUrl })`. + **Key Features**: -- Uses AWS EventBridge Scheduler for reliable one-time job execution +- Uses AWS EventBridge Scheduler for reliable one-time and recurring job execution - Targets SQS queues (accepts `queueUrl`, derives ARN internally) - Mock scheduler available for local development (`SCHEDULER_PROVIDER=mock`) - Auto-cleanup with dead letter queue support diff --git a/docs/architecture-decisions/063-integration-definitions-as-data.md b/docs/architecture-decisions/063-integration-definitions-as-data.md index 6895730c5..c649a5f44 100644 --- a/docs/architecture-decisions/063-integration-definitions-as-data.md +++ b/docs/architecture-decisions/063-integration-definitions-as-data.md @@ -125,7 +125,7 @@ Because a definition can only compose allow-listed, versioned code, a workspace- ## Implementation (v0.1) The decision above was written before any code existed. v0.1 is a working implementation in -`packages/core/integrations/data-defined/`, with 128 tests (the persistence suite runs against real PostgreSQL and +`packages/core/integrations/data-defined/`, with 136 tests (the persistence suite runs against real PostgreSQL and MongoDB). It runs 16 example definitions (Quo ⇄ HubSpot is the reference: seven workflows, one two-way sync, two UI surfaces) against fake API modules, and checks 21 capability manifests. This section records how it works, where it changed the sketch above, and what it does not do yet. File @@ -503,15 +503,56 @@ genuine (HubSpot signs its UI-extension requests) is the renderer's job, not Fri | Moment | What runs | Code | |---|---|---| | **Install** | The customer connects both modules (the existing auth flow), picks settings, and the record is created with the pin to the latest `PUBLISHED` version. `onCreate` loads the definition and runs `lifecycle.onInstall`, for example `quo.webhook.create` pointed at this installation's webhook URL. Every install step with an `id` has its result saved to `config.runtime.installed.` with `patchConfig` | `onCreate` | -| **Configure** | The customer toggles workflows, changes settings or overrides a mapping. `onUpdate` merges config, then recompiles: enabling a workflow adds its event, routes and schedule, and disabling removes them. No deploy, and no other installation is affected | `onUpdate` | +| **Configure** | The customer toggles workflows, changes settings, overrides a mapping or, where the definition allows it, retimes a schedule. `onUpdate` merges config, then recompiles: enabling a workflow adds its event, routes and host schedule, and disabling removes them (§15a). No deploy, and no other installation is affected | `onUpdate` | | **Update** (new version) | Nothing happens to running installations. Each stays on its pin until it is moved, as in §17 | `planUpgrade` | -| **Uninstall** | `onDelete` runs `lifecycle.onUninstall`, which reads what install created (`quo.webhook.delete` with `$.install.runtime.installed.hook.id`; a `foreach` over several webhook ids for Shopify), then the framework removes the record | `onDelete` | +| **Uninstall** | `onDelete` runs `lifecycle.onUninstall`, which reads what install created (`quo.webhook.delete` with `$.install.runtime.installed.hook.id`; a `foreach` over several webhook ids for Shopify), deletes the installation's host schedules, then the framework removes the record | `onDelete` | `$.install` is built by one function, `installContext`, everywhere a definition runs: workflows, queries, UI surfaces, lifecycle steps and option lists. It holds the integration settings with their defaults, `webhookUrl`, `integrationId`, `accounts` and `runtime.installed`. The first cut built it four ways, and the query and UI versions had no `runtime`, so a query reading an install result silently got nothing. +### 15a. Recurring schedules, one per installation + +Schedule-triggered workflows and polling syncs run on a host schedule of their own, one per installation. Recurring +jobs are not specific to data-defined integrations, so the mechanism is in core and a hand-written integration uses +the same one (Quo's crons, for example, which are set by hand per integration today): + +```js +static Definition = { + name: 'housecallpro', + schedules: { + ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // a user may choose >= 15m + DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed + }, +}; +``` + +A data-defined definition says the same thing on the trigger (`{ "type": "schedule", "cron" | "every", "timezone", +"configurable" }`) or on a polling sync (`ongoing: { "mode": "poll", "every", "configurable" }`). The pieces: + +| Layer | What | File | +|---|---|---| +| Recurrence | Host-neutral: `{ every: '15m' }` or a five-field `{ cron, timezone }`. `validateRecurrence` refuses what no host can run: both day fields restricted, a cron that never fires, an unknown zone. `minGapMinutes` measures what an expression really does, so a floor of 15m refuses `0,5 * * * *` | `infrastructure/scheduler/recurrence.js` | +| Port | `upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled })` beside the one-time methods. Each adapter translates the recurrence. The payload may carry `TICK.id` / `TICK.scheduledFor`, which the host fills per firing | `scheduler-service-interface.js` | +| Adapters | EventBridge: `rate(...)` or a six-field `cron(... ? ...)` with weekdays written as names, because EventBridge numbers them 1-7 from Sunday. It updates first, creates when the schedule is missing, and updates again after losing a create race. The tick is filled from ``. Mock: in memory, with `_simulateTrigger(name, tick)` | `eventbridge-scheduler-adapter.js`, `mock-scheduler-adapter.js` | +| Commands | `upsertRecurringJob` and `reconcileRecurringJobs({ desired, stored, queueUrl })`. A job is written only when its recurrence, event, payload or state changed, and deleted when it is no longer wanted. A write the host refuses leaves the stored entry as it was, so the next reconcile retries it | `application/commands/scheduler-commands.js` | +| Desired set | The definition's entries plus the installation's choices. A choice is honoured only where the definition allows it and within its floor. An invalid choice keeps the default and adds a warning message to the integration, so a bad edit never stops a schedule | `integrations/schedules.js` | +| Lifecycle | `IntegrationBase.syncSchedules()` runs from the default `onCreate`, `onUpdate` and `onDelete`. It records what the host holds at `config.runtime.schedules.`. An integration with no schedules does nothing. The data-defined class supplies its entries by compiling the current config, and also calls it after an upgrade | `integration-base.js` | + +The installation's choice lives in its config: `config.schedules.` for a hand-written integration, and +`workflows..schedule` / `syncs..schedule` for a data-defined one, offered in the configure form only where +the definition allows it. Host schedule names are `--`, kept within +EventBridge's 64 characters. Each firing sends `{ event, data: { integrationId, scheduleKey, tick } }` to the +integration's queue, so the existing queue worker hydrates the installation and dispatches the event. A redelivered +tick runs once, because `tick::` is the workflow's dedupe id. Taking configurability away is a +major change. An upgrade keeps a choice only where the new version still allows it, and otherwise reports a +`reschedule` action. + +Ticks for an installation that is not `ENABLED` are discarded by the queue worker, as one-time jobs already are. +Pausing the host schedule when the status changes is not done yet. Changing status does not go through `onUpdate`, +so it needs a hook of its own. + ### 16. UI surfaces A UI extension is a `query` plus a `view`. The view is a tree of a **fixed component set**: `Stack`, `Heading`, @@ -596,7 +637,6 @@ work kept apart from bulk work. | **Sync engine**: backfill pages, ongoing deltas, two-way master and conflict rules | `SYNC_*` events compile, but `runSync` throws "not implemented in v0.1" | Reuse the existing `SYNC_PAGE` and `Process` patterns from `BaseCRMIntegration` | | `delay`, approvals, per-module rate limits | Common in workflows; each needs durable state between steps | With step checkpointing | | Rebuilding links after an identity change | `planUpgrade` reports `relink` / `reindex`; nothing performs it | With the sync engine, which owns backfill | -| Registering `compiled.schedules` | Schedule-triggered workflows and polling syncs compile to `CRON` events, but nothing creates the recurring schedule. Frigg's scheduler commands schedule one-time jobs, so recurring per-installation schedules need either an EventBridge schedule group per installation or one app-level tick that fans out | A design decision of its own | | Host renderers (HubSpot card, Quo panel) | `renderSurface` returns trees that nothing draws yet | Per host | | Real manifests | `capabilities/*.json` are mocks written to match the Api classes | Generated from, or checked against, each module's `Api` | diff --git a/packages/core/application/commands/scheduler-commands.js b/packages/core/application/commands/scheduler-commands.js index 006cf82e7..5ce464083 100644 --- a/packages/core/application/commands/scheduler-commands.js +++ b/packages/core/application/commands/scheduler-commands.js @@ -19,7 +19,7 @@ * }); */ -const { createSchedulerService } = require('../../infrastructure/scheduler'); +const { createSchedulerService, TICK, validateRecurrence, recurrenceKey } = require('../../infrastructure/scheduler'); /** * Derive SQS ARN from SQS URL @@ -89,7 +89,7 @@ function createSchedulerCommands({ integrationName, schedulerService }) { return _schedulerService; } - return { + const commands = { /** * Schedule a one-time job to be executed at a specific time * @@ -179,6 +179,96 @@ function createSchedulerCommands({ integrationName, schedulerService }) { } }, + /** + * Create or update a recurring job. Every firing sends `{ event, data: { ...payload, tick } }` to the queue, + * where `tick` is `{ id, scheduledFor }` for that firing (when the host supplies them), so a handler can + * dedupe a redelivered tick. + * + * @param {Object} params + * @param {string} params.jobId - Unique, stable name for the schedule + * @param {{every?: string, cron?: string, timezone?: string}} params.recurrence - Host-neutral recurrence + * @param {string} params.event - Event name to trigger + * @param {Object} [params.payload] - Data sent on every firing + * @param {string} params.queueUrl - Target SQS queue URL + * @param {boolean} [params.enabled=true] + * @returns {Promise<{jobId, jobArn, created} | {jobId, warning} | {error, reason, code}>} + */ + async upsertRecurringJob({ jobId, recurrence, event, payload, queueUrl, enabled = true }) { + try { + const invalid = (message) => Object.assign(new Error(message), { code: 'INVALID_JOB_DATA' }); + if (!jobId) throw invalid('jobId is required'); + if (!event) throw invalid('event is required'); + if (!queueUrl) throw invalid('queueUrl is required'); + const problems = validateRecurrence(recurrence); + if (problems.length) throw invalid(`Invalid recurrence: ${problems.join('; ')}`); + + const service = getSchedulerService(); + if (!service) { + console.warn(`[${integrationName}] Scheduler not configured, skipping recurring job ${jobId}`); + return { jobId, jobArn: null, warning: 'Scheduler not configured' }; + } + const result = await service.upsertRecurring({ + scheduleName: jobId, + recurrence, + queueResourceId: deriveArnFromQueueUrl(queueUrl), + payload: { + event, + integrationName, + scheduleName: jobId, + data: { ...(payload || {}), tick: { id: TICK.id, scheduledFor: TICK.scheduledFor } }, + }, + enabled, + }); + return { jobId, jobArn: result.scheduledJobId, created: result.created }; + } catch (error) { + console.error(`[${integrationName}] Failed to upsert recurring job ${jobId}:`, error.message); + return mapErrorToResponse(error); + } + }, + + /** + * Make the host's recurring jobs match what an installation should have, and return what to store. + * + * `desired` is the full list for one installation; `stored` is what the last reconcile returned (kept on the + * integration config). A job is upserted when it is new or its recurrence, event, payload or state changed, + * and deleted when it is no longer desired. A failed upsert or delete leaves its stored entry as it was, so + * the next reconcile retries it; nothing is reported as scheduled that the host did not accept. + * + * @param {Object} params + * @param {Array<{key, jobId, recurrence, event, payload?, enabled?}>} params.desired + * @param {Object} [params.stored] + * @param {string} params.queueUrl + * @returns {Promise<{stored, created: string[], updated: string[], deleted: string[], unchanged: string[], failed: Array<{key, reason}>, warning?: string}>} + */ + async reconcileRecurringJobs({ desired = [], stored = {}, queueUrl }) { + const out = { stored: { ...(stored || {}) }, created: [], updated: [], deleted: [], unchanged: [], failed: [] }; + if (!getSchedulerService()) return { ...out, warning: 'Scheduler not configured' }; + const fingerprint = (d) => JSON.stringify([recurrenceKey(d.recurrence), d.event, d.payload || {}, d.enabled !== false, d.jobId]); + const wanted = new Set(); + for (const d of desired) { + wanted.add(d.key); + const prev = out.stored[d.key]; + const print = fingerprint(d); + if (prev && prev.fingerprint === print) { out.unchanged.push(d.key); continue; } + const res = await commands.upsertRecurringJob({ jobId: d.jobId, recurrence: d.recurrence, event: d.event, payload: d.payload, queueUrl, enabled: d.enabled !== false }); + if (res.error || res.warning) { out.failed.push({ key: d.key, reason: res.reason || res.warning }); continue; } + if (prev && prev.jobId !== d.jobId) { + const del = await commands.deleteJob(prev.jobId); + if (del.error) out.failed.push({ key: d.key, reason: `Old schedule ${prev.jobId} not deleted: ${del.reason}` }); + } + out.stored[d.key] = { jobId: d.jobId, recurrence: d.recurrence, event: d.event, enabled: d.enabled !== false, jobArn: res.jobArn, fingerprint: print, updatedAt: new Date().toISOString() }; + (prev ? out.updated : out.created).push(d.key); + } + for (const [key, prev] of Object.entries(out.stored)) { + if (wanted.has(key)) continue; + const del = await commands.deleteJob(prev.jobId); + if (del.error) { out.failed.push({ key, reason: del.reason }); continue; } + delete out.stored[key]; + out.deleted.push(key); + } + return out; + }, + /** * Delete a scheduled job * @@ -256,6 +346,7 @@ function createSchedulerCommands({ integrationName, schedulerService }) { } }, }; + return commands; } module.exports = { diff --git a/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js b/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js new file mode 100644 index 000000000..6e669c11f --- /dev/null +++ b/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js @@ -0,0 +1,228 @@ +/** + * Recurring schedules: the host-neutral recurrence, its EventBridge translation, the adapters' upsert, and the + * reconcile command that keeps one installation's schedules in step with what it should have. + */ +const { + parseEvery, + minGapMinutes, + validateRecurrence, + recurrenceKey, +} = require('../recurrence'); +const { MockSchedulerAdapter } = require('../mock-scheduler-adapter'); +const { TICK, fillTick } = require('../scheduler-service-interface'); +const { createSchedulerCommands } = require('../../../application/commands/scheduler-commands'); + +jest.mock('@aws-sdk/client-scheduler', () => { + class Named extends Error { constructor(m) { super(m); this.name = this.constructor.name; } } + class ResourceNotFoundException extends Named {} + class ConflictException extends Named {} + const command = (type) => class { constructor(input) { this.type = type; this.input = input; } }; + return { + SchedulerClient: class { constructor() { this.sent = []; this.respond = async () => ({ ScheduleArn: 'arn:x' }); } async send(c) { this.sent.push(c); return this.respond(c); } }, + CreateScheduleCommand: command('create'), + UpdateScheduleCommand: command('update'), + DeleteScheduleCommand: command('delete'), + GetScheduleCommand: command('get'), + ResourceNotFoundException, + ConflictException, + }; +}); +const { ResourceNotFoundException, ConflictException } = require('@aws-sdk/client-scheduler'); +const { EventBridgeSchedulerAdapter, toEventBridgeExpression } = require('../eventbridge-scheduler-adapter'); + +const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/hubspot-queue'; +const QUEUE_ARN = 'arn:aws:sqs:us-east-1:123456789012:hubspot-queue'; + +describe('recurrence (host-neutral)', () => { + it('reads every as minutes, hours or days and nothing else', () => { + expect([parseEvery('15m'), parseEvery('2h'), parseEvery('1d'), parseEvery(' 90 m ')]).toEqual([15, 120, 1440, 90]); + for (const bad of ['30s', '0m', '1w', '', null, 'm15', '1.5h']) expect(parseEvery(bad)).toBeNull(); + }); + + it('measures the shortest real gap of a cron, including the wrap into the next hour and day', () => { + expect(minGapMinutes({ cron: '*/15 * * * *' })).toBe(15); + expect(minGapMinutes({ cron: '*/7 * * * *' })).toBe(4); // :56 -> :00 + expect(minGapMinutes({ cron: '5,50 * * * *' })).toBe(15); // :50 -> :05 + expect(minGapMinutes({ cron: '0 */2 * * *' })).toBe(120); + expect(minGapMinutes({ cron: '0 9 * * 1-5' })).toBe(1440); + expect(minGapMinutes({ cron: '0 23,1 * * *' })).toBe(120); // 23:00 -> 01:00 next day + expect(minGapMinutes({ cron: '0 0 * * 0,7' })).toBe(10080); // 0 and 7 are both Sunday + expect(minGapMinutes({ every: '45m' })).toBe(45); + }); + + it('refuses what no host can run: bad fields, both day fields, a cron that never fires, an unknown time zone', () => { + expect(validateRecurrence({ cron: '0 9 * *' })).toEqual([expect.stringMatching(/five fields/)]); + expect(validateRecurrence({ cron: '60 * * * *' })).toEqual([expect.stringMatching(/minute.*out of range/)]); + expect(validateRecurrence({ cron: '0 9 1 * MON' })).toEqual(['Restrict day of month or day of week, not both']); + expect(validateRecurrence({ cron: '0 9 31 2 *' })).toEqual(['"0 9 31 2 *" never fires']); + expect(validateRecurrence({ cron: '0 9 * * *', timezone: 'Mars/Base' })).toEqual(['Unknown time zone "Mars/Base"']); + expect(validateRecurrence({ every: '1h', timezone: 'UTC' })).toEqual(['timezone applies to cron schedules only']); + expect(validateRecurrence({ every: '1h', cron: '0 * * * *' })).toEqual(['Give exactly one of every or cron']); + expect(validateRecurrence({ cron: '0 9 * * 1-5', timezone: 'America/New_York' })).toEqual([]); + }); + + it('holds a schedule to a floor by what it does, not how it is spelled', () => { + expect(validateRecurrence({ every: '10m' }, { min: '15m' })).toEqual(['Runs every 10 minutes at its most frequent; the minimum is 15m']); + expect(validateRecurrence({ cron: '0,5 * * * *' }, { min: '15m' })).toEqual(['Runs every 5 minutes at its most frequent; the minimum is 15m']); + expect(validateRecurrence({ cron: '*/15 * * * *' }, { min: '15m' })).toEqual([]); + expect(validateRecurrence({ every: '1h' }, { min: '1h' })).toEqual([]); + }); + + it('gives equal recurrences equal keys, so a reconcile does not rewrite an unchanged schedule', () => { + expect(recurrenceKey({ every: '60m' })).toBe(recurrenceKey({ every: '1h' })); + expect(recurrenceKey({ cron: '0 9 * * *' })).toBe(recurrenceKey({ cron: '0 9 * * *', timezone: 'UTC' })); + expect(recurrenceKey({ cron: '0 9 * * *', timezone: 'Europe/London' })).not.toBe(recurrenceKey({ cron: '0 9 * * *' })); + }); + + it('fills tick placeholders deeply and leaves everything else alone', () => { + const out = fillTick({ a: TICK.id, b: [TICK.scheduledFor, 'x'], c: { d: TICK.id, e: 1 } }, { id: 'I', scheduledFor: 'T' }); + expect(out).toEqual({ a: 'I', b: ['T', 'x'], c: { d: 'I', e: 1 } }); + }); +}); + +describe('EventBridge translation', () => { + it('uses rate() in the largest whole unit and cron() with six fields, ? and weekday names', () => { + expect(toEventBridgeExpression({ every: '15m' })).toBe('rate(15 minutes)'); + expect(toEventBridgeExpression({ every: '60m' })).toBe('rate(1 hour)'); + expect(toEventBridgeExpression({ every: '2d' })).toBe('rate(2 days)'); + expect(toEventBridgeExpression({ every: '90m' })).toBe('rate(90 minutes)'); + expect(toEventBridgeExpression({ cron: '*/15 * * * *' })).toBe('cron(*/15 * * * ? *)'); + expect(toEventBridgeExpression({ cron: '0 0 1 * *' })).toBe('cron(0 0 1 * ? *)'); + // cron's 1-5 is Monday-Friday; EventBridge would read 1-5 as Sunday-Thursday, so names are written. + expect(toEventBridgeExpression({ cron: '0 9 * * 1-5' })).toBe('cron(0 9 ? * MON,TUE,WED,THU,FRI *)'); + expect(toEventBridgeExpression({ cron: '0 8 * JAN-MAR 0' })).toBe('cron(0 8 ? JAN-MAR SUN *)'); + }); + + it('throws on a recurrence it cannot express rather than creating a wrong schedule', () => { + expect(() => toEventBridgeExpression({ cron: '0 9 1 * MON' })).toThrow(/not both/); + }); +}); + +describe('EventBridgeSchedulerAdapter.upsertRecurring', () => { + let adapter; + beforeEach(() => { + process.env.SCHEDULER_ROLE_ARN = 'arn:aws:iam::123456789012:role/scheduler'; + adapter = new EventBridgeSchedulerAdapter({ region: 'us-east-1' }); + }); + + const recurring = { scheduleName: 'hubspot-int1-sync', recurrence: { cron: '0 9 * * 1-5', timezone: 'America/New_York' }, queueResourceId: QUEUE_ARN, payload: { event: 'SYNC', data: { tick: { id: TICK.id, scheduledFor: TICK.scheduledFor } } } }; + + it('updates an existing schedule: expression, zone, state, and per-firing context attributes in the input', async () => { + const out = await adapter.upsertRecurring({ ...recurring, enabled: false }); + expect(out).toEqual({ scheduledJobId: 'arn:x', created: false }); + expect(adapter.client.sent.map((c) => c.type)).toEqual(['update']); + const input = adapter.client.sent[0].input; + expect(input).toMatchObject({ Name: 'hubspot-int1-sync', ScheduleExpression: 'cron(0 9 ? * MON,TUE,WED,THU,FRI *)', ScheduleExpressionTimezone: 'America/New_York', State: 'DISABLED', FlexibleTimeWindow: { Mode: 'OFF' } }); + expect(input.ActionAfterCompletion).toBeUndefined(); // a recurring schedule must not delete itself + expect(JSON.parse(input.Target.Input).data.tick).toEqual({ id: '', scheduledFor: '' }); + }); + + it('creates it when it does not exist', async () => { + adapter.client.respond = async (c) => { if (c.type === 'update') throw new ResourceNotFoundException('nope'); return { ScheduleArn: 'arn:new' }; }; + expect(await adapter.upsertRecurring(recurring)).toEqual({ scheduledJobId: 'arn:new', created: true }); + expect(adapter.client.sent.map((c) => c.type)).toEqual(['update', 'create']); + expect(adapter.client.sent[1].input.State).toBe('ENABLED'); + }); + + it('updates after losing a create race to a concurrent writer', async () => { + let updates = 0; + adapter.client.respond = async (c) => { + if (c.type === 'update' && updates++ === 0) throw new ResourceNotFoundException('nope'); + if (c.type === 'create') throw new ConflictException('exists'); + return { ScheduleArn: 'arn:raced' }; + }; + expect(await adapter.upsertRecurring(recurring)).toEqual({ scheduledJobId: 'arn:raced', created: false }); + expect(adapter.client.sent.map((c) => c.type)).toEqual(['update', 'create', 'update']); + }); + + it('rate schedules always run in UTC; other errors propagate', async () => { + await adapter.upsertRecurring({ ...recurring, recurrence: { every: '1h' } }); + expect(adapter.client.sent[0].input).toMatchObject({ ScheduleExpression: 'rate(1 hour)', ScheduleExpressionTimezone: 'UTC' }); + adapter.client.respond = async () => { throw new Error('AccessDenied'); }; + await expect(adapter.upsertRecurring(recurring)).rejects.toThrow('AccessDenied'); + }); +}); + +describe('scheduler commands: recurring jobs', () => { + let service, commands; + const desired = (overrides = {}) => ({ key: 'SYNC', jobId: 'hubspot-int1-sync', recurrence: { every: '1h' }, event: 'SYNC', payload: { integrationId: 'int1' }, ...overrides }); + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + jest.spyOn(console, 'warn').mockImplementation(() => {}); + service = new MockSchedulerAdapter(); + commands = createSchedulerCommands({ integrationName: 'hubspot', schedulerService: service }); + }); + afterEach(() => jest.restoreAllMocks()); + + it('sends { event, data: { ...payload, tick } } to the queue ARN on every firing', async () => { + const res = await commands.upsertRecurringJob({ jobId: 'j1', recurrence: { every: '15m' }, event: 'POLL', payload: { integrationId: 'int1' }, queueUrl: QUEUE_URL }); + expect(res).toMatchObject({ jobId: 'j1', created: true }); + const stored = service._getSchedules().j1; + expect(stored.queueResourceId).toBe(QUEUE_ARN); + expect(service._simulateTrigger('j1', { id: 'exec-1', scheduledFor: '2026-10-18T09:00:00Z' })).toEqual({ + event: 'POLL', integrationName: 'hubspot', scheduleName: 'j1', + data: { integrationId: 'int1', tick: { id: 'exec-1', scheduledFor: '2026-10-18T09:00:00Z' } }, + }); + }); + + it('refuses an invalid recurrence as INVALID_JOB_DATA (400) without touching the host', async () => { + const res = await commands.upsertRecurringJob({ jobId: 'j1', recurrence: { every: '30s' }, event: 'POLL', queueUrl: QUEUE_URL }); + expect(res).toMatchObject({ error: 400, code: 'INVALID_JOB_DATA' }); + expect(service._getSchedules()).toEqual({}); + }); + + it('reconcile: creates, leaves an unchanged schedule alone, updates a changed one, deletes a dropped one', async () => { + const first = await commands.reconcileRecurringJobs({ desired: [desired(), desired({ key: 'DIGEST', jobId: 'hubspot-int1-digest', recurrence: { cron: '0 8 * * *' }, event: 'DIGEST' })], stored: {}, queueUrl: QUEUE_URL }); + expect(first).toMatchObject({ created: ['SYNC', 'DIGEST'], updated: [], deleted: [], failed: [] }); + expect(first.stored.SYNC).toMatchObject({ jobId: 'hubspot-int1-sync', recurrence: { every: '1h' }, event: 'SYNC', enabled: true }); + + const upsert = jest.spyOn(service, 'upsertRecurring'); + const second = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '60m' } })], stored: first.stored, queueUrl: QUEUE_URL }); + expect(second).toMatchObject({ unchanged: ['SYNC'], deleted: ['DIGEST'], created: [], updated: [] }); + expect(upsert).not.toHaveBeenCalled(); // 60m is 1h: nothing to write + expect(Object.keys(service._getSchedules())).toEqual(['hubspot-int1-sync']); + + const third = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '30m' }, enabled: false })], stored: second.stored, queueUrl: QUEUE_URL }); + expect(third.updated).toEqual(['SYNC']); + expect(service._getSchedules()['hubspot-int1-sync']).toMatchObject({ recurrence: { every: '30m' }, state: 'DISABLED' }); + expect(third.stored).toEqual({ SYNC: expect.objectContaining({ recurrence: { every: '30m' }, enabled: false }) }); + }); + + it('reconcile: a failed write leaves the stored entry as it was, so the next reconcile retries it', async () => { + const first = await commands.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); + jest.spyOn(service, 'upsertRecurring').mockRejectedValueOnce(new Error('throttled')); + const second = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '2h' } })], stored: first.stored, queueUrl: QUEUE_URL }); + expect(second.failed).toEqual([{ key: 'SYNC', reason: 'throttled' }]); + expect(second.stored).toEqual(first.stored); + const third = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '2h' } })], stored: second.stored, queueUrl: QUEUE_URL }); + expect(third.updated).toEqual(['SYNC']); + + jest.spyOn(service, 'deleteSchedule').mockRejectedValueOnce(new Error('denied')); + const fourth = await commands.reconcileRecurringJobs({ desired: [], stored: third.stored, queueUrl: QUEUE_URL }); + expect(fourth.failed).toEqual([{ key: 'SYNC', reason: 'denied' }]); + expect(fourth.stored).toEqual(third.stored); // still recorded: the host still holds it + }); + + it('reconcile: a renamed job is created under the new name and the old one deleted', async () => { + const first = await commands.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); + const second = await commands.reconcileRecurringJobs({ desired: [desired({ jobId: 'hubspot-int1-sync-v2' })], stored: first.stored, queueUrl: QUEUE_URL }); + expect(second.updated).toEqual(['SYNC']); + expect(Object.keys(service._getSchedules())).toEqual(['hubspot-int1-sync-v2']); + }); + + it('reconcile without a scheduler records nothing and says so', async () => { + const unconfigured = createSchedulerCommands({ integrationName: 'hubspot' }); + process.env.SCHEDULER_PROVIDER = 'nope'; + try { + const out = await unconfigured.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); + expect(out).toMatchObject({ warning: 'Scheduler not configured', stored: {}, created: [] }); + } finally { delete process.env.SCHEDULER_PROVIDER; } + }); + + it('the methods work detached from the commands object', async () => { + const { reconcileRecurringJobs } = commands; + const out = await reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); + expect(out.created).toEqual(['SYNC']); + }); +}); diff --git a/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js b/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js index c06c46375..63b988908 100644 --- a/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js +++ b/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js @@ -14,12 +14,37 @@ const { SchedulerClient, CreateScheduleCommand, + UpdateScheduleCommand, DeleteScheduleCommand, GetScheduleCommand, ResourceNotFoundException, + ConflictException, } = require('@aws-sdk/client-scheduler'); -const { SchedulerServiceInterface } = require('./scheduler-service-interface'); +const { SchedulerServiceInterface, TICK, fillTick } = require('./scheduler-service-interface'); +const { parseEvery, parseCron, validateRecurrence } = require('./recurrence'); + +const DAY_NAMES = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']; + +/** + * Host-neutral recurrence -> EventBridge Scheduler expression. + * { every: '15m' } -> rate(15 minutes) (whole hours and days use those units) + * { cron: 'm h dom mon dow' } -> cron(m h dom mon dow *), with `?` in whichever day field is unrestricted and the + * weekdays written as names, because EventBridge numbers them 1-7 from Sunday where cron numbers 0-6. + */ +function toEventBridgeExpression(recurrence) { + const problems = validateRecurrence(recurrence); + if (problems.length) throw new Error(`Invalid recurrence: ${problems.join('; ')}`); + if (recurrence.every !== undefined) { + const n = parseEvery(recurrence.every); + const [count, unit] = n % 1440 === 0 ? [n / 1440, 'day'] : n % 60 === 0 ? [n / 60, 'hour'] : [n, 'minute']; + return `rate(${count} ${unit}${count === 1 ? '' : 's'})`; + } + const parsed = parseCron(recurrence.cron); + const [minute, hour, dom, month] = parsed.parts; + const dow = parsed.restricted.dow ? [...parsed.fields[4]].map((d) => DAY_NAMES[d]).join(',') : '?'; + return `cron(${minute} ${hour} ${parsed.restricted.dow ? '?' : dom} ${month} ${dow} *)`; +} class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { constructor({ region } = {}) { @@ -96,6 +121,45 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { } } + /** + * Create or update a recurring schedule that sends `payload` to the queue on every firing. + * Update first (the common case once installed), create when it does not exist, and update again if a concurrent + * writer created it in between. + */ + async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled = true }) { + if (!scheduleName) throw new Error('scheduleName is required'); + if (!queueResourceId) throw new Error('queueResourceId is required'); + if (!this.roleArn) throw new Error('SCHEDULER_ROLE_ARN environment variable is not set'); + + const input = { + Name: scheduleName, + GroupName: this.scheduleGroupName, + ScheduleExpression: toEventBridgeExpression(recurrence), + ScheduleExpressionTimezone: recurrence.every !== undefined ? 'UTC' : recurrence.timezone || 'UTC', + FlexibleTimeWindow: { Mode: 'OFF' }, + State: enabled ? 'ENABLED' : 'DISABLED', + Target: { + Arn: queueResourceId, + RoleArn: this.roleArn, + // EventBridge substitutes these context attributes per invocation. + Input: JSON.stringify(fillTick(payload, { id: '', scheduledFor: '' })), + }, + }; + const update = async () => (await this.client.send(new UpdateScheduleCommand(input))).ScheduleArn; + try { + return { scheduledJobId: await update(), created: false }; + } catch (error) { + if (!(error instanceof ResourceNotFoundException)) throw error; + } + try { + const response = await this.client.send(new CreateScheduleCommand(input)); + return { scheduledJobId: response.ScheduleArn, created: true }; + } catch (error) { + if (error instanceof ConflictException) return { scheduledJobId: await update(), created: false }; + throw error; + } + } + /** * Delete a schedule * @@ -165,6 +229,8 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { exists: true, scheduledAt, state: response.State, + expression: response.ScheduleExpression, + timezone: response.ScheduleExpressionTimezone, }; } catch (error) { if (error instanceof ResourceNotFoundException) { @@ -181,4 +247,4 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { } } -module.exports = { EventBridgeSchedulerAdapter }; +module.exports = { EventBridgeSchedulerAdapter, toEventBridgeExpression, TICK }; diff --git a/packages/core/infrastructure/scheduler/index.js b/packages/core/infrastructure/scheduler/index.js index bbffae59b..904573ee2 100644 --- a/packages/core/infrastructure/scheduler/index.js +++ b/packages/core/infrastructure/scheduler/index.js @@ -1,7 +1,8 @@ /** * Scheduler Infrastructure * - * Provides scheduling capabilities for one-time jobs. + * Provides scheduling capabilities for one-time and recurring jobs. Recurrence is host-neutral (recurrence.js); + * each adapter translates it. * Follows hexagonal architecture with interface + adapters pattern. * * Providers: @@ -9,7 +10,8 @@ * - mock: In-memory mock scheduler (local development) */ -const { SchedulerServiceInterface } = require('./scheduler-service-interface'); +const { SchedulerServiceInterface, TICK, fillTick } = require('./scheduler-service-interface'); +const recurrence = require('./recurrence'); const { EventBridgeSchedulerAdapter } = require('./eventbridge-scheduler-adapter'); const { MockSchedulerAdapter } = require('./mock-scheduler-adapter'); const { @@ -21,6 +23,9 @@ const { module.exports = { // Interface (Port) SchedulerServiceInterface, + TICK, + fillTick, + ...recurrence, // Adapters EventBridgeSchedulerAdapter, diff --git a/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js b/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js index 9826e4934..51970be77 100644 --- a/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js +++ b/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js @@ -7,7 +7,8 @@ * This adapter implements SchedulerServiceInterface for local development and testing. */ -const { SchedulerServiceInterface } = require('./scheduler-service-interface'); +const { SchedulerServiceInterface, fillTick } = require('./scheduler-service-interface'); +const { validateRecurrence } = require('./recurrence'); class MockSchedulerAdapter extends SchedulerServiceInterface { constructor(options = {}) { @@ -61,6 +62,28 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { }; } + /** + * Create or update a recurring schedule (kept in memory; `_simulateTrigger` fires it). + */ + async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled = true }) { + if (!scheduleName) throw new Error('scheduleName is required'); + if (!queueResourceId) throw new Error('queueResourceId is required'); + const problems = validateRecurrence(recurrence); + if (problems.length) throw new Error(`Invalid recurrence: ${problems.join('; ')}`); + const created = !this.schedules.has(scheduleName); + this.schedules.set(scheduleName, { + scheduleName, + recurrence, + queueResourceId, + payload, + recurring: true, + createdAt: this.schedules.get(scheduleName)?.createdAt || new Date().toISOString(), + state: enabled ? 'ENABLED' : 'DISABLED', + }); + if (this.verbose) console.log(`[MockScheduler] ${created ? 'Created' : 'Updated'} recurring schedule: ${scheduleName}`, recurrence); + return { scheduledJobId: `mock-job-${scheduleName}`, created }; + } + /** * Delete a scheduled job * @@ -99,6 +122,7 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { exists: true, scheduledAt: schedule.scheduledAt, state: schedule.state, + ...(schedule.recurrence ? { recurrence: schedule.recurrence } : {}), }; } @@ -126,7 +150,7 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { * @param {string} scheduleName - Name of the schedule to trigger * @returns {Object|null} The payload that would be sent, or null if not found */ - _simulateTrigger(scheduleName) { + _simulateTrigger(scheduleName, tick = { id: `mock-${Date.now()}`, scheduledFor: new Date().toISOString() }) { const schedule = this.schedules.get(scheduleName); if (!schedule) { console.log(`[MockScheduler] Cannot trigger - schedule not found: ${scheduleName}`); @@ -134,9 +158,10 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { } console.log(`[MockScheduler] Simulating trigger for: ${scheduleName}`); - console.log(`[MockScheduler] Payload:`, JSON.stringify(schedule.payload, null, 2)); + const payload = schedule.recurring ? fillTick(schedule.payload, tick) : schedule.payload; + console.log(`[MockScheduler] Payload:`, JSON.stringify(payload, null, 2)); - return schedule.payload; + return payload; } } diff --git a/packages/core/infrastructure/scheduler/recurrence.js b/packages/core/infrastructure/scheduler/recurrence.js new file mode 100644 index 000000000..4cccf95f4 --- /dev/null +++ b/packages/core/infrastructure/scheduler/recurrence.js @@ -0,0 +1,169 @@ +/** + * Recurrence: how often a recurring job fires, in a form no host owns. + * + * { every: '15m' } every 15 minutes ('30s' is refused: hosts tick in minutes) + * { cron: '0 9 * * 1-5', timezone: 'America/New_York' } standard five-field cron (minute hour dom month dow) + * + * Each scheduler adapter translates this into its host's syntax (EventBridge: rate(...) / cron(...) with six fields + * and `?`). Keeping the neutral form here is what lets one definition, or one integration, run on any host. + * + * Pure; no I/O. + */ + +const EVERY = /^(\d+)\s*(m|h|d)$/; +const UNIT_MINUTES = { m: 1, h: 60, d: 1440 }; + +/** '15m' -> 15, '2h' -> 120, '1d' -> 1440. Anything else -> null. */ +function parseEvery(every) { + const m = EVERY.exec(String(every ?? '').trim()); + if (!m) return null; + const n = Number(m[1]) * UNIT_MINUTES[m[2]]; + return n > 0 ? n : null; +} + +const MONTHS = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']; +const DAYS = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']; +const FIELDS = [ + { name: 'minute', min: 0, max: 59 }, + { name: 'hour', min: 0, max: 23 }, + { name: 'day of month', min: 1, max: 31 }, + { name: 'month', min: 1, max: 12, names: MONTHS, base: 1 }, + { name: 'day of week', min: 0, max: 7, names: DAYS, base: 0 }, // 0 and 7 are both Sunday +]; + +function atom(token, field) { + const t = token.toUpperCase(); + if (field.names) { + const i = field.names.indexOf(t); + if (i >= 0) return i + field.base; + } + if (!/^\d+$/.test(t)) return null; + const n = Number(t); + return n >= field.min && n <= field.max ? n : null; +} + +/** One cron field -> the sorted set of values it matches, or an error string. */ +function parseField(text, field) { + const out = new Set(); + for (const part of text.split(',')) { + const [range, stepText] = part.split('/'); + if (part.split('/').length > 2) return `${field.name}: "${part}" has more than one step`; + const step = stepText === undefined ? 1 : Number(stepText); + if (!Number.isInteger(step) || step < 1) return `${field.name}: step "${stepText}" is not a positive whole number`; + let lo, hi; + if (range === '*') { lo = field.min; hi = field.name === 'day of week' ? 6 : field.max; } + else if (range.includes('-')) { + const [a, b] = range.split('-'); + lo = atom(a, field); hi = atom(b, field); + if (lo === null || hi === null) return `${field.name}: "${range}" is out of range`; + if (lo > hi) return `${field.name}: "${range}" runs backwards`; + } else { + lo = atom(range, field); + if (lo === null) return `${field.name}: "${range}" is out of range`; + hi = stepText === undefined ? lo : (field.name === 'day of week' ? 6 : field.max); + } + for (let v = lo; v <= hi; v += step) out.add(field.name === 'day of week' && v === 7 ? 0 : v); + } + return [...out].sort((a, b) => a - b); +} + +/** + * Parse a five-field cron. Returns { fields: [Set x5], restricted: { dom, dow } } or { error }. + * `restricted` records whether day-of-month / day-of-week were given (not `*`), which decides how they combine. + */ +function parseCron(cron) { + const parts = String(cron ?? '').trim().split(/\s+/); + if (parts.length !== 5 || parts[0] === '') return { error: 'A cron expression has five fields: minute hour day-of-month month day-of-week' }; + const fields = []; + for (let i = 0; i < 5; i += 1) { + const v = parseField(parts[i], FIELDS[i]); + if (typeof v === 'string') return { error: v }; + fields.push(new Set(v)); + } + return { fields, parts, restricted: { dom: parts[2] !== '*', dow: parts[4] !== '*' } }; +} + +/** Does a parsed cron fire at this (UTC-free, wall-clock) minute? Standard cron: dom and dow OR when both are set. */ +function matches(parsed, d) { + const [mi, h, dom, mon, dow] = parsed.fields; + if (!mi.has(d.getUTCMinutes()) || !h.has(d.getUTCHours()) || !mon.has(d.getUTCMonth() + 1)) return false; + const domHit = dom.has(d.getUTCDate()), dowHit = dow.has(d.getUTCDay()); + if (parsed.restricted.dom && parsed.restricted.dow) return domHit || dowHit; + return domHit && dowHit; +} + +/** + * The shortest gap, in minutes, between two firings. For `every` it is the interval; for cron it is measured over + * fifteen months of wall-clock minutes (long enough to see every month and weekday combination), so a floor such as + * "no more often than every 15 minutes" is checked against what the expression really does, not its spelling. + * Returns Infinity for an expression that fires at most once in that window, null when invalid. + */ +function minGapMinutes(recurrence) { + if (recurrence?.every !== undefined) return parseEvery(recurrence.every); + const parsed = parseCron(recurrence?.cron); + return parsed.error ? null : scan(parsed).best; +} + +/** Walk fifteen months of firings: how many there are and the shortest gap between two. */ +function scan(parsed) { + const [mi, h] = parsed.fields; + const minutes = [...mi], hours = [...h]; + let prev = null, best = Infinity, count = 0; + const start = Date.UTC(2027, 0, 1); + for (let day = 0; day < 457 && best > 1; day += 1) { // fifteen months: every month and weekday pairing + const midnight = start + day * 86400000; + if (!matches(parsed, new Date(midnight + hours[0] * 3600000 + minutes[0] * 60000))) continue; + for (const hr of hours) { + for (const mn of minutes) { + const t = day * 1440 + hr * 60 + mn; + if (prev !== null && t - prev < best) best = t - prev; + prev = t; + count += 1; + } + } + } + return { best, count }; +} + +/** Is `tz` an IANA time zone this runtime knows? */ +function isTimeZone(tz) { + try { new Intl.DateTimeFormat('en-US', { timeZone: tz }); return true; } catch { return false; } +} + +/** + * Validate a recurrence. `min` is an optional floor ('15m'): the expression must never fire more often. + * @returns {string[]} problems, empty when valid + */ +function validateRecurrence(recurrence, { min } = {}) { + const problems = []; + if (!recurrence || typeof recurrence !== 'object') return ['Give a schedule as { every } or { cron, timezone? }']; + const hasEvery = recurrence.every !== undefined, hasCron = recurrence.cron !== undefined; + if (hasEvery === hasCron) return ['Give exactly one of every or cron']; + if (hasEvery && parseEvery(recurrence.every) === null) problems.push(`every "${recurrence.every}" is not a whole number of minutes, hours or days (15m, 2h, 1d)`); + if (hasCron) { + const parsed = parseCron(recurrence.cron); + if (parsed.error) problems.push(parsed.error); + // EventBridge (and Quartz) cannot express "this day of month OR this weekday"; refuse it everywhere, so a + // schedule that validates here runs on every host. + else if (parsed.restricted.dom && parsed.restricted.dow) problems.push('Restrict day of month or day of week, not both'); + else if (scan(parsed).count === 0) problems.push(`"${recurrence.cron}" never fires`); + } + if (recurrence.timezone !== undefined && (hasEvery || !isTimeZone(recurrence.timezone))) { + problems.push(hasEvery ? 'timezone applies to cron schedules only' : `Unknown time zone "${recurrence.timezone}"`); + } + if (!problems.length && min !== undefined) { + const floor = parseEvery(min); + const gap = minGapMinutes(recurrence); + if (floor === null) problems.push(`minimum interval "${min}" is not a duration (15m, 2h, 1d)`); + else if (gap !== null && gap < floor) problems.push(`Runs every ${gap} minute${gap === 1 ? '' : 's'} at its most frequent; the minimum is ${min}`); + } + return problems; +} + +/** The comparable identity of a recurrence (what decides whether a stored schedule must be updated). */ +function recurrenceKey(r) { + if (!r) return ''; + return r.every !== undefined ? `every:${parseEvery(r.every)}` : `cron:${String(r.cron).trim().split(/\s+/).join(' ')}@${r.timezone || 'UTC'}`; +} + +module.exports = { parseEvery, parseCron, minGapMinutes, validateRecurrence, recurrenceKey, isTimeZone, matches }; diff --git a/packages/core/infrastructure/scheduler/scheduler-service-interface.js b/packages/core/infrastructure/scheduler/scheduler-service-interface.js index 5972593b8..5220f2246 100644 --- a/packages/core/infrastructure/scheduler/scheduler-service-interface.js +++ b/packages/core/infrastructure/scheduler/scheduler-service-interface.js @@ -23,6 +23,25 @@ class SchedulerServiceInterface { throw new Error('Method scheduleOneTime must be implemented by subclass'); } + /** + * Create or update a recurring schedule (idempotent by name). + * + * The recurrence is host-neutral ({ every } or { cron, timezone }, see recurrence.js); each adapter translates it. + * String values in `payload` equal to TICK.id or TICK.scheduledFor are replaced, per firing, with that firing's + * id and scheduled time when the host can supply them, so a consumer can dedupe redeliveries of one tick. + * + * @param {Object} params + * @param {string} params.scheduleName - Unique name for the schedule + * @param {{every?: string, cron?: string, timezone?: string}} params.recurrence + * @param {string} params.queueResourceId - Queue resource identifier to send message to + * @param {Object} params.payload - JSON payload to send on every firing + * @param {boolean} [params.enabled=true] - false keeps the schedule but stops it firing + * @returns {Promise<{scheduledJobId: string, created: boolean}>} + */ + async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled }) { + throw new Error('Method upsertRecurring must be implemented by subclass'); + } + /** * Delete a scheduled job * @@ -44,4 +63,19 @@ class SchedulerServiceInterface { } } -module.exports = { SchedulerServiceInterface }; +/** Placeholders an adapter fills per firing (see upsertRecurring). */ +const TICK = Object.freeze({ id: '{{frigg.tick.id}}', scheduledFor: '{{frigg.tick.scheduledFor}}' }); + +/** Replace every string equal to a TICK placeholder, deeply. */ +function fillTick(payload, values) { + const swap = (v) => { + if (v === TICK.id) return values.id; + if (v === TICK.scheduledFor) return values.scheduledFor; + if (Array.isArray(v)) return v.map(swap); + if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, swap(x)])); + return v; + }; + return swap(payload); +} + +module.exports = { SchedulerServiceInterface, TICK, fillTick }; diff --git a/packages/core/integrations/data-defined/README.md b/packages/core/integrations/data-defined/README.md index 1569233d1..5d89ae4c0 100644 --- a/packages/core/integrations/data-defined/README.md +++ b/packages/core/integrations/data-defined/README.md @@ -96,6 +96,20 @@ different document at the same version is refused. So is a version that is not n semver bump smaller than the changes. Publishing goes through a conditional write, so two concurrent publishes of one version cannot both win. With `memoryDefinitions()` (definitions bundled in the app) the admin routes answer 501. +## Schedules + +A schedule-triggered workflow or a polling sync gets one recurring host schedule per installation, through the same +core mechanism hand-written integrations use (`Definition.schedules`; ADR-063 §15a). Mark one `configurable` to let +installations pick their own timing, optionally with a floor: + +```json +"trigger": { "type": "schedule", "cron": "0 16 * * 1-5", "timezone": "America/New_York", "configurable": { "min": "1h" } } +"ongoing": { "mode": "poll", "every": "15m", "configurable": true } +``` + +The choice goes in `workflows..schedule` / `syncs..schedule` (the configure form offers the field). Create, +update, upgrade and delete keep the host in step. What it holds is recorded at `config.runtime.schedules`. + ## Steps (v0.1) Built-ins: `links.find`, `links.upsert`, `map`, `merge`, `template`, `set`, `stop`, `foreach` (≤1000), `branch`, `code`. @@ -164,6 +178,4 @@ CLI as a child process; it fails, rather than skips, if `ajv` is missing. Core's the existing `SYNC_PAGE` pattern. `runSync` throws until then. - `delay` / wait steps, approvals that pause a run, and per-tenant rate limits. - The host renderers (the HubSpot app extension and Quo panel that draw `renderSurface` output). -- Registering `compiled.schedules` as recurring schedules (Frigg's scheduler commands are one-time jobs). They compile - today; nothing creates them yet. - Rebuilding `link:` / `idx:` rows after an identity change; `planUpgrade` reports it as `relink` / `reindex`. diff --git a/packages/core/integrations/data-defined/compile.js b/packages/core/integrations/data-defined/compile.js index ba9d4a98d..eefbcecff 100644 --- a/packages/core/integrations/data-defined/compile.js +++ b/packages/core/integrations/data-defined/compile.js @@ -17,7 +17,8 @@ function isOn(item, chosen) { function compile(def, installation = {}) { const events = {}; const webhooks = {}; // moduleKey -> topic -> [eventName] - const schedules = []; // { event, cron, timezone } + const schedules = []; // { key, event, default: { every } | { cron, timezone }, configurable, choice } (integrations/schedules.js) + const configurableOf = (c) => (c ? (c === true ? {} : c) : false); const userActions = []; // { event, label, from } const routes = []; // { method, path, event } const add = (name, e) => { events[name] = e; return name; }; @@ -39,7 +40,7 @@ function compile(def, installation = {}) { } } else if (mode === 'poll') { const n = add(eventName('SYNC', s.id, 'ONGOING'), { type: 'CRON', ref: { kind: 'sync', id: s.id, phase: 'ongoing' } }); - schedules.push({ event: n, every: s.ongoing.every || '15m' }); + schedules.push({ key: n, event: n, default: { every: s.ongoing.every || '15m' }, configurable: configurableOf(s.ongoing.configurable), choice: installation.syncs?.[s.id]?.schedule }); } } @@ -49,7 +50,10 @@ function compile(def, installation = {}) { const type = { webhook: 'WEBHOOK', schedule: 'CRON', userAction: 'USER_ACTION', syncRecord: 'QUEUE' }[t.type]; const n = add(eventName('WF', w.id), { type, ...(type === 'USER_ACTION' ? { userActionType: 'DATA_ACTION', title: t.label } : {}), ref: { kind: 'workflow', id: w.id } }); if (t.type === 'webhook') route(t.module, t.topic, n); - if (t.type === 'schedule') schedules.push({ event: n, cron: t.cron, timezone: t.timezone || 'UTC' }); + if (t.type === 'schedule') { + const recurrence = t.every !== undefined ? { every: t.every } : { cron: t.cron, ...(t.timezone ? { timezone: t.timezone } : {}) }; + schedules.push({ key: n, event: n, default: recurrence, configurable: configurableOf(t.configurable), choice: installation.workflows?.[w.id]?.schedule }); + } if (t.type === 'userAction') userActions.push({ event: n, label: t.label, from: t.from || null }); if (t.type === 'syncRecord') route('$sync', `${t.sync}:${(t.on || []).join('|')}`, n); } diff --git a/packages/core/integrations/data-defined/config-form.js b/packages/core/integrations/data-defined/config-form.js index d82bc9575..e43f1f256 100644 --- a/packages/core/integrations/data-defined/config-form.js +++ b/packages/core/integrations/data-defined/config-form.js @@ -38,6 +38,20 @@ async function settingsSchema(settings, pathPrefix, values, options, integration return { schema: { type: 'object', properties, ...(required.length ? { required } : {}) }, ui }; } +/** + * The timing field of a schedule the definition marks configurable: an interval when its default is an interval, + * a cron (with time zone) when its default is a cron. The floor is said in the help text and enforced on save. + */ +function scheduleField(recurrence, configurable) { + if (!configurable || !recurrence) return null; + const min = configurable.min ? ` No more often than every ${configurable.min}.` : ''; + if (recurrence.every !== undefined) { + return { schema: { type: 'object', title: 'Schedule', properties: { every: { type: 'string', title: 'Runs every', pattern: '^\\d+\\s*(m|h|d)$', default: recurrence.every } } }, ui: { every: { 'ui:help': `Minutes, hours or days: 15m, 2h, 1d.${min}` } } }; + } + return { schema: { type: 'object', title: 'Schedule', properties: { cron: { type: 'string', title: 'Cron', default: recurrence.cron }, timezone: { type: 'string', title: 'Time zone', default: recurrence.timezone || 'UTC' } } }, ui: { cron: { 'ui:help': `minute hour day-of-month month day-of-week.${min}` } } }; +} +const configurableOf = (c) => (c ? (c === true ? {} : c) : false); + const toggle = (title, description, enabledByDefault) => ({ type: 'boolean', title, ...(description ? { description } : {}), default: enabledByDefault }); const isOn = (item, chosen) => (typeof chosen?.enabled === 'boolean' ? chosen.enabled : (item.default || 'on') === 'on'); @@ -63,6 +77,12 @@ async function buildConfigForm(def, config = {}, options = async () => []) { for (const s of def.syncs) { jsonSchema.properties.syncs.properties[s.id] = { type: 'object', title: s.label, properties: { enabled: toggle('On', s.description, (s.default || 'on') === 'on') } }; data.syncs[s.id] = { enabled: isOn(s, config.syncs?.[s.id]) }; + const field = s.ongoing?.mode === 'poll' && scheduleField({ every: s.ongoing.every || '15m' }, configurableOf(s.ongoing.configurable)); + if (field) { + jsonSchema.properties.syncs.properties[s.id].properties.schedule = field.schema; + ((uiSchema.syncs ||= {})[s.id] ||= {}).schedule = field.ui; + if (config.syncs?.[s.id]?.schedule) data.syncs[s.id].schedule = { ...config.syncs[s.id].schedule }; + } } } @@ -81,6 +101,13 @@ async function buildConfigForm(def, config = {}, options = async () => []) { // Settings only matter while the workflow is on: required only then. node.dependencies = { enabled: { oneOf: [{ properties: { enabled: { const: false } } }, { properties: { enabled: { const: true } }, required: ['settings'] }] } }; } + const t = w.trigger || {}; + const field = t.type === 'schedule' && scheduleField(t.every !== undefined ? { every: t.every } : { cron: t.cron, timezone: t.timezone }, configurableOf(t.configurable)); + if (field) { + node.properties.schedule = field.schema; + (uiSchema.workflows[w.id] ||= {}).schedule = field.ui; + if (chosen.schedule) data.workflows[w.id].schedule = { ...chosen.schedule }; + } jsonSchema.properties.workflows.properties[w.id] = node; } } diff --git a/packages/core/integrations/data-defined/data-defined-integration.js b/packages/core/integrations/data-defined/data-defined-integration.js index cf0c5023c..1d7e10dbe 100644 --- a/packages/core/integrations/data-defined/data-defined-integration.js +++ b/packages/core/integrations/data-defined/data-defined-integration.js @@ -150,7 +150,11 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati ? await definitions.get(this.def.id, msg.definitionVersion) || this.def : this.def; const installation = this.installation(); - if (ref.kind === 'workflow') return this.recordRun(await runWorkflow({ def, installation, workflowId: ref.id, event: msg.body ?? msg, eventId: msg.eventId, record: msg.record, ports: this.ports, manifests })); + // A scheduled tick may be delivered twice; its scheduled time makes the dedupe id (an unfilled + // placeholder, from a host that cannot supply it, is not an id). + const scheduledFor = msg.tick?.scheduledFor && !String(msg.tick.scheduledFor).startsWith('{{') ? msg.tick.scheduledFor : null; + const eventId = msg.eventId || (scheduledFor ? `tick:${msg.scheduleKey}:${scheduledFor}` : undefined); + if (ref.kind === 'workflow') return this.recordRun(await runWorkflow({ def, installation, workflowId: ref.id, event: msg.body ?? msg, eventId, record: msg.record, ports: this.ports, manifests })); if (ref.kind === 'query') return runQuery({ def, installation, queryId: ref.id, record: msg.record, ports: this.ports }); if (ref.kind === 'ui') return renderSurface({ def, installation, surfaceId: ref.id, record: payload.req?.body?.record || msg.record, ports: this.ports }); if (ref.kind === 'sync') return this.runSync(ref, msg); @@ -252,6 +256,17 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati return super.onDelete(params); } + /** + * The recurring schedules this installation should have: the definition's schedule-triggered workflows and + * polling syncs that are switched on, with the installation's timing where the definition allows it. + * Compiled from the current config every time, so the base class's onUpdate (which patches config, then + * reconciles) schedules what was just chosen. The base class's onCreate, onUpdate and onDelete keep the host + * in step (IntegrationBase.syncSchedules). + */ + scheduleEntries() { + return this.def ? compile(this.def, this.config || {}).schedules : []; + } + /** The configure form (GET /api/integrations/:id/config/options), generated from the definition. */ async getConfigOptions() { return buildConfigForm(this.def, this.config || {}, (path, values, integrationSettings) => this.optionsFor(path, values, integrationSettings)); @@ -316,12 +331,14 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati await runLifecycle({ def: this.def, installation: this.installation(), hook: 'onUninstall', ports: this.ports }); const next = await definitions.get(plan.config.definition.id, plan.config.definition.version); const out = await runLifecycle({ def: next, installation: { ...this.installation(), ...config }, hook: 'onInstall', ports: this.portsFor(next) }); - config = { ...config, runtime: { installed: Object.fromEntries((next.lifecycle?.onInstall || []).filter((st) => st.id).map((st) => [st.id, out.steps?.[st.id] ?? null])) } }; + config = { ...config, runtime: { ...(this.config?.runtime || {}), installed: Object.fromEntries((next.lifecycle?.onInstall || []).filter((st) => st.id).map((st) => [st.id, out.steps?.[st.id] ?? null])) } }; } await this.patchConfig(config); this.config = { ...this.config, ...config }; await this.loadDefinition(); - return { ...plan, config, applied: true }; + // The new version may add, drop or retime schedules. + const schedules = typeof this.syncSchedules === 'function' ? await this.syncSchedules() : null; + return { ...plan, config, applied: true, ...(schedules ? { schedules } : {}) }; } async recordRun(run) { diff --git a/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json b/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json index a0e05cfd0..cf93da323 100644 --- a/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json +++ b/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json @@ -40,6 +40,9 @@ "mapping": { "type": "array", "description": "Full replacement mapping when the customer overrides the default." + }, + "schedule": { + "$ref": "#/$defs/scheduleChoice" } } } @@ -54,6 +57,9 @@ }, "settings": { "type": "object" + }, + "schedule": { + "$ref": "#/$defs/scheduleChoice" } } } @@ -71,13 +77,91 @@ }, "runtime": { "type": "object", - "description": "Owned by the runtime, not the customer. installed: each onInstall step's result by step id (webhook subscriptions), read by onUninstall as $.install.runtime.installed.. Sync cursors join it with the sync engine.", + "description": "Owned by the runtime, not the customer. installed: each onInstall step's result by step id (webhook subscriptions), read by onUninstall as $.install.runtime.installed.. schedules: the recurring schedules the host holds for this installation, by event (written by IntegrationBase.syncSchedules). Sync cursors join it with the sync engine.", "additionalProperties": false, "properties": { "installed": { "type": "object" + }, + "schedules": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "jobId", + "recurrence", + "event" + ], + "properties": { + "jobId": { + "type": "string" + }, + "recurrence": { + "type": "object" + }, + "event": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "jobArn": { + "type": [ + "string", + "null" + ] + }, + "fingerprint": { + "type": "string" + }, + "updatedAt": { + "type": "string" + } + } + } } } } + }, + "$defs": { + "scheduleChoice": { + "description": "The installation's timing for a schedule the definition marks configurable: every, or cron with an optional time zone. Honoured only within the definition's minimum; otherwise the definition's schedule runs.", + "type": "object", + "additionalProperties": false, + "properties": { + "every": { + "type": "string", + "pattern": "^\\d+\\s*(m|h|d)$" + }, + "cron": { + "type": "string" + }, + "timezone": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "every" + ], + "not": { + "required": [ + "cron" + ] + } + }, + { + "required": [ + "cron" + ], + "not": { + "required": [ + "every" + ] + } + } + ] + } } } diff --git a/packages/core/integrations/data-defined/schema/integration-definition.v0.1.schema.json b/packages/core/integrations/data-defined/schema/integration-definition.v0.1.schema.json index 4620b8577..c30f9d4fa 100644 --- a/packages/core/integrations/data-defined/schema/integration-definition.v0.1.schema.json +++ b/packages/core/integrations/data-defined/schema/integration-definition.v0.1.schema.json @@ -217,7 +217,12 @@ ] }, "every": { - "type": "string" + "type": "string", + "pattern": "^\\d+\\s*(m|h|d)$", + "description": "Poll interval (default 15m)." + }, + "configurable": { + "$ref": "#/$defs/scheduleConfigurable" } } }, @@ -662,8 +667,7 @@ { "type": "object", "required": [ - "type", - "cron" + "type" ], "additionalProperties": false, "properties": { @@ -671,12 +675,44 @@ "const": "schedule" }, "cron": { - "type": "string" + "type": "string", + "description": "Five-field cron (minute hour day-of-month month day-of-week). Restrict day of month or day of week, not both." }, "timezone": { - "type": "string" + "type": "string", + "description": "IANA time zone for cron (default UTC)." + }, + "every": { + "type": "string", + "pattern": "^\\d+\\s*(m|h|d)$", + "description": "Fixed interval (15m, 2h, 1d). Give cron or every." + }, + "configurable": { + "$ref": "#/$defs/scheduleConfigurable" } - } + }, + "oneOf": [ + { + "required": [ + "cron" + ], + "not": { + "required": [ + "every" + ] + } + }, + { + "required": [ + "every" + ], + "not": { + "required": [ + "cron" + ] + } + } + ] }, { "type": "object", @@ -843,6 +879,24 @@ } } ] + }, + "scheduleConfigurable": { + "description": "Lets each installation choose its own timing. true: any valid timing. { min }: never more often than this (15m, 2h, 1d).", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "min": { + "type": "string", + "pattern": "^\\d+\\s*(m|h|d)$" + } + } + } + ] } } } diff --git a/packages/core/integrations/data-defined/tests/schedules.test.js b/packages/core/integrations/data-defined/tests/schedules.test.js new file mode 100644 index 000000000..5ef0ed89d --- /dev/null +++ b/packages/core/integrations/data-defined/tests/schedules.test.js @@ -0,0 +1,178 @@ +/** + * Recurring schedules for data-defined installations: what a definition may declare, what an installation may + * change, and that the class keeps one host schedule per installation in step through create, update, upgrade and + * delete (IntegrationBase.syncSchedules over the real scheduler commands and the mock adapter). + */ +jest.mock('../../../database/config', () => ({ DB_TYPE: 'mongodb', getDatabaseType: () => 'mongodb', PRISMA_LOG_LEVEL: 'error,warn' })); + +const d = require('..'); +const { compile } = require('../compile'); +const { validate } = require('../validate'); +const { diff, planUpgrade } = require('../versioning'); +const { buildConfigForm } = require('../config-form'); +const { IntegrationBase } = require('../../integration-base'); +const { MockSchedulerAdapter } = require('../../../infrastructure/scheduler/mock-scheduler-adapter'); +const { createSchedulerCommands } = require('../../../application/commands/scheduler-commands'); + +const manifests = d.loadManifests(); +const example = (id) => JSON.parse(JSON.stringify(d.loadExamples().find((x) => x.id === id))); +const DIGEST = 'WF_DUE_TOMORROW_DIGEST'; + +/** asana-slack with its digest made configurable (never more often than hourly). */ +function digestDef({ configurable = { min: '1h' }, version } = {}) { + const def = example('asana-slack'); + const w = def.workflows.find((x) => x.id === 'due-tomorrow-digest'); + if (configurable !== null) w.trigger.configurable = configurable; // null: leave it fixed + if (version) def.version = version; + return def; +} + +describe('definition and installation', () => { + it('compiles each switched-on schedule into an entry with its default, its allowance and the installation\'s choice', () => { + const def = digestDef(); + const out = compile(def, { workflows: { 'due-tomorrow-digest': { schedule: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } } } }); + expect(out.schedules).toEqual([{ key: DIGEST, event: DIGEST, default: { cron: '0 16 * * 1-5', timezone: 'America/New_York' }, configurable: { min: '1h' }, choice: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } }]); + expect(compile(def, { workflows: { 'due-tomorrow-digest': { enabled: false } } }).schedules).toEqual([]); + }); + + it('a polling sync compiles to an interval entry, configurable when the definition says so', () => { + const def = example('pipedrive-hubspot'); + def.syncs[0].ongoing = { mode: 'poll', every: '30m', configurable: true }; + const out = compile(def, { syncs: { [def.syncs[0].id]: { schedule: { every: '2h' } } } }); + expect(out.schedules).toEqual([expect.objectContaining({ default: { every: '30m' }, configurable: {}, choice: { every: '2h' } })]); + }); + + it('validates a schedule as a host would run it, and a default that breaks its own floor', () => { + const errors = (def) => validate(def, manifests).errors.filter((e) => /trigger|ongoing/.test(e.path)).map((e) => `${e.path}: ${e.message}`); + const w = (def) => def.workflows.find((x) => x.id === 'due-tomorrow-digest'); + let def = digestDef(); w(def).trigger.cron = '0 9 1 * 1'; + expect(errors(def)).toEqual(['workflows[0].trigger.cron: Restrict day of month or day of week, not both']); + def = digestDef(); w(def).trigger.every = '1h'; + expect(errors(def)).toEqual(['workflows[0].trigger: Give exactly one of cron or every']); + def = digestDef({ configurable: { min: '2d' } }); + expect(errors(def)).toEqual(['workflows[0].trigger.configurable.min: The default schedule breaks its own minimum: Runs every 1440 minutes at its most frequent; the minimum is 2d']); + def = example('pipedrive-hubspot'); def.syncs[0].ongoing = { mode: 'webhook', configurable: true }; + expect(validate(def, manifests).errors.map((e) => e.message)).toContain('Only a polling sync has an interval to configure'); + expect(d.checkDefinition(digestDef(), manifests).valid).toBe(true); // and the JSON Schema accepts the new fields + }); + + it('offers a timing field in the configure form only where the definition allows it', async () => { + const configurable = await buildConfigForm(digestDef(), { workflows: { 'due-tomorrow-digest': { schedule: { cron: '0 9 * * *' } } } }); + expect(configurable.jsonSchema.properties.workflows.properties['due-tomorrow-digest'].properties.schedule.properties).toEqual({ + cron: { type: 'string', title: 'Cron', default: '0 16 * * 1-5' }, timezone: { type: 'string', title: 'Time zone', default: 'America/New_York' }, + }); + expect(configurable.data.workflows['due-tomorrow-digest'].schedule).toEqual({ cron: '0 9 * * *' }); + expect(configurable.uiSchema.workflows['due-tomorrow-digest'].schedule.cron['ui:help']).toMatch(/No more often than every 1h/); + const fixed = await buildConfigForm(digestDef({ configurable: null }), {}); + expect(fixed.jsonSchema.properties.workflows.properties['due-tomorrow-digest'].properties.schedule).toBeUndefined(); + }); + + it('versioning: taking configurability away is major; an upgrade keeps a choice only where the new version allows it', () => { + expect(diff(digestDef(), digestDef({ configurable: false, version: '2.0.0' })).changes).toContainEqual(expect.objectContaining({ level: 'major', path: 'workflows.due-tomorrow-digest.trigger.configurable' })); + expect(diff(digestDef({ configurable: false }), digestDef({ version: '1.1.0' })).level).toBe('minor'); + + const install = { definition: { id: 'asana-slack', version: '1.0.0' }, workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '2h' } } } }; + const tighter = planUpgrade(install, digestDef(), digestDef({ configurable: { min: '4h' }, version: '1.1.0' })); + expect(tighter.config.workflows['due-tomorrow-digest'].schedule).toBeUndefined(); + expect(tighter.actions).toContainEqual(expect.objectContaining({ kind: 'reschedule', id: 'workflows.due-tomorrow-digest' })); + const same = planUpgrade(install, digestDef(), digestDef({ version: '1.1.0' })); + expect(same.config.workflows['due-tomorrow-digest'].schedule).toEqual({ every: '2h' }); + expect(same.actions.map((a) => a.kind)).not.toContain('reschedule'); + }); +}); + +describe('installation lifecycle', () => { + let service; + + /** IntegrationBase's real lifecycle and syncSchedules, with persistence and module wiring stubbed. */ + class Base extends IntegrationBase { + constructor(p = {}) { + super(p); + this.id = p.id; this.config = p.config; this.status = 'IN_CREATION'; + this.updateIntegrationStatus = { execute: async () => true }; + this.updateIntegrationMessages = { execute: jest.fn(async () => true) }; + this.patchIntegrationConfig = { execute: async (id, patch) => ({ config: (this.config = { ...this.config, ...patch }) }) }; + this.schedulerCommands = createSchedulerCommands({ integrationName: 'data-defined', schedulerService: service }); + this.ran = []; + } + async initialize() { this.registerEventHandlers(); } + registerEventHandlers() {} + async getMapping() { return null; } + async upsertMapping() {} + async createMapping(k) { if (this.ran.includes(k)) { const e = new Error('exists'); e.name = 'MappingAlreadyExistsError'; throw e; } this.ran.push(k); } + async deleteMapping() {} + } + + const setup = (defs) => { + const definitions = d.memoryDefinitions(defs); + const Cls = d.defineSharedIntegrationClass({ definitions, manifests, modules: { asana: {}, slack: {} }, IntegrationBase: Base, runs: false }); + return (config) => { const inst = new Cls({ id: 'int9', config: { type: 'data-defined', ...config } }); const api = new Proxy({}, { get: () => async () => ({ id: 'X1' }) }); // install steps succeed + inst.asana = { api }; inst.slack = { api }; return inst; }; + }; + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + process.env.DATA_DEFINED_QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/data-defined-queue'; + service = new MockSchedulerAdapter(); + }); + afterEach(() => { jest.restoreAllMocks(); delete process.env.DATA_DEFINED_QUEUE_URL; }); + + it('create schedules the installation\'s workflows; an update to its own timing retimes it; delete removes it', async () => { + const newInstall = setup([digestDef()]); + const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version } }); + await inst.initialize(); + await inst.onCreate(); + const name = 'data-defined-int9-wf_due_tomorrow_digest'; + expect(service._getSchedules()[name]).toMatchObject({ recurrence: { cron: '0 16 * * 1-5', timezone: 'America/New_York' }, state: 'ENABLED' }); + expect(inst.config.runtime.schedules[DIGEST]).toMatchObject({ jobId: name, event: DIGEST }); + + await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '3h' } } } } }); + expect(service._getSchedules()[name].recurrence).toEqual({ every: '3h' }); + + await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '10m' } } } } }); + expect(service._getSchedules()[name].recurrence).toEqual({ cron: '0 16 * * 1-5', timezone: 'America/New_York' }); + expect(inst.updateIntegrationMessages.execute).toHaveBeenCalledWith('int9', 'warnings', 'Schedule not changed', `${DIGEST}: Runs every 10 minutes at its most frequent; the minimum is 1h`, expect.any(Number)); + + await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: false } } } }); + expect(service._getSchedules()).toEqual({}); // switching the workflow off removes its schedule + + await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true } } } }); + await inst.onDelete(); + expect(service._getSchedules()).toEqual({}); + expect(inst.config.runtime.schedules).toEqual({}); + }); + + it('an upgrade retimes schedules and keeps the install steps\' results beside them', async () => { + const v2 = digestDef({ configurable: false, version: '2.0.0' }); + v2.workflows.find((x) => x.id === 'due-tomorrow-digest').trigger.cron = '0 15 * * 1-5'; + const newInstall = setup([digestDef(), v2]); + const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version }, settings: { workspace: 'W1', project: 'P1' }, workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '2h' } }, 'completed-in-project': { enabled: true, settings: { channel: 'C1' } } } }); + await inst.initialize(); + await inst.onCreate(); + expect(service._getSchedules()['data-defined-int9-wf_due_tomorrow_digest'].recurrence).toEqual({ every: '2h' }); + + const out = await inst.upgradeTo('2.0.0', { apply: true }); + expect(out.applied).toBe(true); + expect(out.actions).toContainEqual(expect.objectContaining({ kind: 'reschedule' })); + expect(service._getSchedules()['data-defined-int9-wf_due_tomorrow_digest'].recurrence).toEqual({ cron: '0 15 * * 1-5', timezone: 'America/New_York' }); + expect(inst.config.runtime.schedules[DIGEST].recurrence).toEqual({ cron: '0 15 * * 1-5', timezone: 'America/New_York' }); + }); + + it('a redelivered tick runs once: the tick\'s scheduled time is the dedupe id', async () => { + const newInstall = setup([digestDef()]); + const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version } }); + await inst.initialize(); + const runs = []; + inst.recordRun = async (r) => { runs.push(r); return r; }; + const tick = { event: DIGEST, data: { integrationId: 'int9', scheduleKey: DIGEST, tick: { id: 'exec-1', scheduledFor: '2026-10-19T20:00:00Z' } } }; + await inst.events[DIGEST].handler(tick); + await inst.events[DIGEST].handler(tick); + expect(runs.map((r) => r.status)).toEqual([expect.not.stringMatching(/duplicate/), 'duplicate']); + expect(runs[0].trigger.eventId).toBe(`tick:${DIGEST}:2026-10-19T20:00:00Z`); + // An unfilled placeholder (a host that cannot supply the time) is not an id, so it cannot block every tick. + const unfilled = { data: { scheduleKey: DIGEST, tick: { scheduledFor: '{{frigg.tick.scheduledFor}}' } } }; + await inst.events[DIGEST].handler(unfilled); + await inst.events[DIGEST].handler(unfilled); + expect(runs.slice(2).map((r) => r.status)).not.toContain('duplicate'); + }); +}); diff --git a/packages/core/integrations/data-defined/validate.js b/packages/core/integrations/data-defined/validate.js index 191ac1885..e7f4b3aa4 100644 --- a/packages/core/integrations/data-defined/validate.js +++ b/packages/core/integrations/data-defined/validate.js @@ -13,6 +13,7 @@ const { BUILTINS, isBuiltin, splitCapability } = require('./steps'); const { parseCondition, referencedPaths, templateTags, FILTERS } = require('./expressions'); const { TRANSFORMS: TRANSFORM_FNS } = require('./transforms'); +const { validateRecurrence } = require('../../infrastructure/scheduler/recurrence'); const ID = /^[a-z][a-z0-9-]*$/; const SEMVER = /^\d+\.\d+\.\d+(-[\w.]+)?$/; @@ -26,6 +27,14 @@ function validate(def, manifests = {}) { const errors = [], warnings = []; const err = (path, message) => errors.push({ path, message }); const warn = (path, message) => warnings.push({ path, message }); + /** A schedule's own timing must be valid, and must itself respect the floor it sets for installations. */ + const checkSchedule = (recurrence, configurable, path) => { + const where = recurrence.every !== undefined ? `${path}.every` : `${path}.cron`; + for (const m of validateRecurrence(recurrence)) err(where, m); + if (configurable && typeof configurable === 'object' && configurable.min !== undefined) { + for (const m of validateRecurrence(recurrence, { min: configurable.min })) if (!validateRecurrence(recurrence).includes(m)) err(`${path}.configurable.min`, `The default schedule breaks its own minimum: ${m}`); + } + }; if (!def || typeof def !== 'object') return { valid: false, errors: [{ path: '', message: 'Definition must be an object' }], warnings }; if (def.schemaVersion !== '0.1') err('schemaVersion', 'Must be "0.1"'); @@ -154,6 +163,8 @@ function validate(def, manifests = {}) { const hasEvents = (k, o) => Object.values(manifestOf[k]?.events || {}).some((e) => e.object === o); if (manifestOf[src.k] && !hasEvents(src.k, src.o)) warn(`${p}.ongoing`, `${s.source} declares no webhook events; ongoing sync will need polling`); } + if (s.ongoing?.mode === 'poll') checkSchedule({ every: s.ongoing.every || '15m' }, s.ongoing.configurable, `${p}.ongoing`); + else if (s.ongoing?.configurable) err(`${p}.ongoing.configurable`, 'Only a polling sync has an interval to configure'); if (s.filter) checkCondition(s.filter, `${p}.filter`); }); @@ -273,7 +284,10 @@ function validate(def, manifests = {}) { if (!manifestOf[t.module]) err(`${p}.trigger.module`, `Unknown module key "${t.module}"`); else if (!manifestOf[t.module].events?.[t.topic]) err(`${p}.trigger.topic`, `Module "${t.module}" emits no event "${t.topic}"`); } - if (t.type === 'schedule' && String(t.cron || '').trim().split(/\s+/).length !== 5) err(`${p}.trigger.cron`, 'Five-field cron expression'); + if (t.type === 'schedule') { + if ((t.cron === undefined) === (t.every === undefined)) err(`${p}.trigger`, 'Give exactly one of cron or every'); + else checkSchedule(t.every !== undefined ? { every: t.every } : { cron: t.cron, ...(t.timezone ? { timezone: t.timezone } : {}) }, t.configurable, `${p}.trigger`); + } if (t.type === 'userAction' && t.from && !uis[t.from]) err(`${p}.trigger.from`, `Unknown UI surface "${t.from}"`); if (t.type === 'syncRecord' && !syncs[t.sync]) err(`${p}.trigger.sync`, `Unknown sync "${t.sync}"`); if (t.filter) checkCondition(t.filter, `${p}.trigger.filter`); diff --git a/packages/core/integrations/data-defined/versioning.js b/packages/core/integrations/data-defined/versioning.js index 5d895c6e9..0ce3fe795 100644 --- a/packages/core/integrations/data-defined/versioning.js +++ b/packages/core/integrations/data-defined/versioning.js @@ -51,7 +51,8 @@ function diff(from, to) { if (o.direction !== s.direction) push('major', `syncs.${id}.direction`, 'Sync direction changed'); if (!same(o.identity, s.identity)) push('major', `syncs.${id}.identity`, 'Identity keys changed: existing links must be rebuilt'); if (!same(o.mapping, s.mapping)) push('patch', `syncs.${id}.mapping`, 'Default mapping changed (installations with their own mapping keep it)'); - if (!same(o.ongoing, s.ongoing) || !same(o.backfill, s.backfill) || o.filter !== s.filter) push('minor', `syncs.${id}`, 'Sync schedule or filter changed'); + if (o.ongoing?.configurable && !s.ongoing?.configurable) push('major', `syncs.${id}.ongoing.configurable`, 'Poll interval no longer configurable: installations lose their chosen interval'); + else if (!same(o.ongoing, s.ongoing) || !same(o.backfill, s.backfill) || o.filter !== s.filter) push('minor', `syncs.${id}`, 'Sync schedule or filter changed'); } const la = byId(from.links), lb = byId(to.links); @@ -70,6 +71,7 @@ function diff(from, to) { const o = wa[id]; if (!o) { push('minor', `workflows.${id}`, `New workflow (default ${w.default || 'on'})`); continue; } if (o.trigger?.type !== w.trigger?.type || o.trigger?.module !== w.trigger?.module) push('major', `workflows.${id}.trigger`, 'Trigger source changed'); + else if (o.trigger?.configurable && !w.trigger?.configurable) push('major', `workflows.${id}.trigger.configurable`, 'Schedule no longer configurable: installations lose their chosen timing'); else if (!same(o.trigger, w.trigger)) push('minor', `workflows.${id}.trigger`, 'Trigger topic, schedule or filter changed'); diffSettings(o.settings, w.settings, `workflows.${id}.settings`, push); if (!same(o.steps, w.steps)) push('patch', `workflows.${id}.steps`, 'Steps changed'); @@ -152,6 +154,16 @@ function planUpgrade(installation, from, to) { const la = byId(from.links), lb = byId(to.links); for (const l of to.links || []) if (la[l.id] && !same(la[l.id].identity, l.identity)) actions.push({ kind: 'reindex', id: l.id, message: `${l.label || l.id}: links by id still resolve; lookups by the new identity keys miss until each pair is linked again` }); for (const id of Object.keys(la)) if (!lb[id]) actions.push({ kind: 'drop-links', id, message: `Link set "${la[id].label || id}" is gone; its stored links are no longer read` }); + // An installation's own timing survives an upgrade only where the new version still allows it. + const keepTiming = (chosen, allowed, path, label) => { + if (!chosen?.schedule) return; + const problems = !allowed ? ['not configurable'] : validateRecurrence(recurrenceOf(chosen.schedule), { min: allowed === true ? undefined : allowed.min }); + if (!problems.length) return; + delete chosen.schedule; + actions.push({ kind: 'reschedule', id: path, message: `${label}: back to the definition's schedule (${problems.join('; ')})` }); + }; + for (const w of to.workflows || []) if (w.trigger?.type === 'schedule') keepTiming(config.workflows?.[w.id], w.trigger.configurable, `workflows.${w.id}`, w.label || w.id); + for (const sy of to.syncs || []) keepTiming(config.syncs?.[sy.id], sy.ongoing?.mode === 'poll' && sy.ongoing.configurable, `syncs.${sy.id}`, sy.label || sy.id); if (!same(from.lifecycle?.onInstall, to.lifecycle?.onInstall)) actions.push({ kind: 'reinstall', message: 'Re-run install steps (webhook subscriptions)' }); actions.push({ kind: 'drain', message: `Messages already queued under ${from.version} finish on ${from.version}; new events run on ${to.version}` }); @@ -159,3 +171,6 @@ function planUpgrade(installation, from, to) { } module.exports = { diff, checkBump, planUpgrade, bumpOf }; + +const { validateRecurrence } = require('../../infrastructure/scheduler/recurrence'); +const { recurrenceOf } = require('../schedules'); diff --git a/packages/core/integrations/integration-base.js b/packages/core/integrations/integration-base.js index 8a0f95b5c..9a293e1d5 100644 --- a/packages/core/integrations/integration-base.js +++ b/packages/core/integrations/integration-base.js @@ -22,6 +22,7 @@ const { getTelemetry } = require('../telemetry/telemetry-runtime'); const { instrumentHandler } = require('../telemetry/instrument-handler'); const { bindTelemetryContext } = require('../telemetry/bind-telemetry-context'); const { getLogger } = require('../logs'); +const { desiredSchedules, entriesFromDefinition } = require('./schedules'); const constantsToBeMigrated = { defaultEvents: { @@ -500,6 +501,7 @@ class IntegrationBase { async onCreate() { const needsConfig = await this.validateConfig(); await this.persistStatus(needsConfig ? 'NEEDS_CONFIG' : 'ENABLED'); + await this.syncSchedules(); } /** @@ -522,9 +524,69 @@ class IntegrationBase { } else if (this.status === 'NEEDS_CONFIG') { await this.persistStatus('ENABLED'); } + await this.syncSchedules(); } - async onDelete(params) {} + async onDelete(params) { + await this.syncSchedules({ remove: true }); + } + + /** + * The recurring schedules this installation should have (integrations/schedules.js). The default reads + * `Definition.schedules` and the installation's choices at `config.schedules`; data-defined integrations + * override it with their compiled definition. + * @returns {Array} schedule entries + */ + scheduleEntries() { + return entriesFromDefinition(this.constructor.Definition?.schedules, this.config || {}); + } + + /** + * Make the host's recurring schedules match `scheduleEntries()`, one per installation, and record what the host + * holds at `config.runtime.schedules`. Called by the default onCreate, onUpdate and onDelete; an override of + * those calls it itself. Does nothing for an integration that declares no schedules and has none recorded. + * + * Never throws: a schedule that could not be written is logged, reported in the result and left recorded as it + * was, and the next call retries it. A choice the definition does not allow keeps the default schedule and adds + * a warning message to the integration. + * + * @param {{ remove?: boolean }} [options] remove: delete every recorded schedule (uninstall) + * @returns {Promise} the reconcile outcome, or null when there was nothing to do + */ + async syncSchedules({ remove = false } = {}) { + const stored = this.config?.runtime?.schedules || {}; + const entries = remove ? [] : this.scheduleEntries() || []; + if (!entries.length && !Object.keys(stored).length) return null; + if (this.id === undefined || this.id === null) return null; + const name = this.constructor.Definition?.name; + try { + const { desired, problems } = desiredSchedules({ integrationName: name, integrationId: this.id, entries }); + for (const p of problems) { + await this.updateIntegrationMessages.execute(this.id, 'warnings', 'Schedule not changed', `${p.key}: ${p.problems.join('; ')}`, Date.now()); + } + const commands = typeof this.schedulerCommands?.reconcileRecurringJobs === 'function' + ? this.schedulerCommands + : require('../application/commands/scheduler-commands').createSchedulerCommands({ integrationName: name }); + const outcome = await commands.reconcileRecurringJobs({ desired, stored, queueUrl: this.scheduleQueueUrl() }); + if (outcome.warning) return { ...outcome, problems }; + if (JSON.stringify(outcome.stored) !== JSON.stringify(stored)) { + await this.patchConfig({ runtime: { ...(this.config?.runtime || {}), schedules: outcome.stored } }); + } + if (outcome.failed.length) { + this.logger.error('Schedules not reconciled', { eventName: `${this.logger.name}.schedules_failed`, failed: outcome.failed }); + } + return { ...outcome, problems }; + } catch (error) { + this.logger.error('Schedules not reconciled', { eventName: `${this.logger.name}.schedules_failed`, error }); + return { error: error.message }; + } + } + + /** The queue a schedule fires into: this integration's queue (`_QUEUE_URL`). */ + scheduleQueueUrl() { + if (typeof this.getQueueUrl === 'function') return this.getQueueUrl(); + return process.env[`${this.constructor.Definition.name.toUpperCase().replace(/-/g, '_')}_QUEUE_URL`]; + } async getConfigOptions() { const options = { diff --git a/packages/core/integrations/schedules.js b/packages/core/integrations/schedules.js new file mode 100644 index 000000000..d013e4ad0 --- /dev/null +++ b/packages/core/integrations/schedules.js @@ -0,0 +1,89 @@ +/** + * Recurring schedules per installation: which ones an integration should have right now. Pure; no I/O. + * + * A developer declares them; an installation may change the timing of one only when the developer allowed it: + * + * static Definition = { + * schedules: { + * ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // the user may pick any interval >= 15m + * DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed + * }, + * } + * + * The key is the event the schedule sends. An installation's choice lives at `config.schedules.` (`{ every }`, + * `{ cron, timezone }`, or `{ enabled: false }` when configurable); what the host actually holds is recorded by the + * runtime at `config.runtime.schedules.` (see scheduler-commands reconcileRecurringJobs). + * + * Data-defined integrations build the same entries from their compiled definition, so both forms share one + * reconcile path and one storage shape. + */ +const { createHash } = require('crypto'); +const { validateRecurrence } = require('../infrastructure/scheduler/recurrence'); + +const recurrenceOf = (r) => (r?.every !== undefined ? { every: r.every } : r?.cron !== undefined ? { cron: r.cron, ...(r.timezone ? { timezone: r.timezone } : {}) } : null); + +/** + * A stable host schedule name: `--`, within the 64 characters EventBridge allows (and only + * the characters it allows). Longer names keep a readable prefix and end in a short hash, so they stay unique. + */ +function scheduleJobId(integrationName, integrationId, key) { + const clean = (v) => String(v).replace(/[^0-9A-Za-z_.-]/g, '-'); + const full = `${clean(integrationName)}-${clean(integrationId)}-${clean(key).toLowerCase()}`; + if (full.length <= 64) return full; + const hash = createHash('sha256').update(full).digest('hex').slice(0, 10); + return `${full.slice(0, 53)}-${hash}`; +} + +/** + * @param {Object} params + * @param {string} params.integrationName + * @param {string|number} params.integrationId + * @param {Array<{key, event?, default: {every?|cron?, timezone?}, configurable?: false|{min?}, choice?: object, enabled?: boolean, payload?: object}>} params.entries + * default: the developer's recurrence. choice: the installation's override (honoured only when configurable). + * @returns {{ desired: Array<{key, jobId, event, recurrence, payload, enabled, source}>, problems: Array<{key, problems: string[]}> }} + * An entry whose choice is invalid keeps its default and is reported, so a bad edit never stops a schedule. + */ +function desiredSchedules({ integrationName, integrationId, entries = [] }) { + const desired = [], problems = []; + for (const e of entries) { + let recurrence = recurrenceOf(e.default), source = 'definition', enabled = e.enabled !== false; + const choice = e.choice && typeof e.choice === 'object' ? e.choice : null; + if (choice && (recurrenceOf(choice) || choice.enabled !== undefined)) { + if (!e.configurable) problems.push({ key: e.key, problems: ['This schedule is not configurable'] }); + else { + const picked = recurrenceOf(choice); + const issues = picked ? validateRecurrence(picked, { min: e.configurable.min }) : []; + if (issues.length) problems.push({ key: e.key, problems: issues }); + else { + if (picked) { recurrence = picked; source = 'installation'; } + if (choice.enabled === false) enabled = false; + } + } + } + const own = validateRecurrence(recurrence); + if (own.length) { problems.push({ key: e.key, problems: own.map((p) => `default: ${p}`) }); continue; } + desired.push({ + key: e.key, + jobId: scheduleJobId(integrationName, integrationId, e.key), + event: e.event || e.key, + recurrence, + payload: { integrationId, scheduleKey: e.key, ...(e.payload || {}) }, + enabled, + source, + }); + } + return { desired, problems }; +} + +/** The entries a hand-written integration declares: Definition.schedules plus the installation's choices. */ +function entriesFromDefinition(definitionSchedules = {}, config = {}) { + return Object.entries(definitionSchedules || {}).map(([key, s]) => ({ + key, + event: s.event || key, + default: s, + configurable: s.configurable ? (s.configurable === true ? {} : s.configurable) : false, + choice: config?.schedules?.[key], + })); +} + +module.exports = { desiredSchedules, entriesFromDefinition, scheduleJobId, recurrenceOf }; diff --git a/packages/core/integrations/tests/integration-base-schedules.test.js b/packages/core/integrations/tests/integration-base-schedules.test.js new file mode 100644 index 000000000..fee77b3ed --- /dev/null +++ b/packages/core/integrations/tests/integration-base-schedules.test.js @@ -0,0 +1,137 @@ +jest.mock('../../database/config', () => ({ + DB_TYPE: 'mongodb', + getDatabaseType: jest.fn(() => 'mongodb'), + PRISMA_LOG_LEVEL: 'error,warn', +})); + +const { IntegrationBase } = require('../integration-base'); +const { desiredSchedules, scheduleJobId } = require('../schedules'); +const { MockSchedulerAdapter } = require('../../infrastructure/scheduler/mock-scheduler-adapter'); +const { createSchedulerCommands } = require('../../application/commands/scheduler-commands'); + +const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/poller-queue'; + +describe('desiredSchedules', () => { + const entry = (over = {}) => ({ key: 'POLL', default: { every: '1h' }, configurable: { min: '15m' }, ...over }); + + it('uses the default, or the installation\'s choice where the definition allows it', () => { + expect(desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry()] }).desired).toEqual([ + { key: 'POLL', jobId: 'poller-i1-poll', event: 'POLL', recurrence: { every: '1h' }, payload: { integrationId: 'i1', scheduleKey: 'POLL' }, enabled: true, source: 'definition' }, + ]); + const chosen = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ choice: { cron: '0 9 * * 1-5', timezone: 'Europe/London' } })] }); + expect(chosen.desired[0]).toMatchObject({ recurrence: { cron: '0 9 * * 1-5', timezone: 'Europe/London' }, source: 'installation' }); + expect(chosen.problems).toEqual([]); + }); + + it('keeps the default and reports it when a choice breaks the floor or the schedule is fixed', () => { + const tooFast = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ choice: { every: '5m' } })] }); + expect(tooFast.desired[0]).toMatchObject({ recurrence: { every: '1h' }, source: 'definition' }); + expect(tooFast.problems).toEqual([{ key: 'POLL', problems: ['Runs every 5 minutes at its most frequent; the minimum is 15m'] }]); + const fixed = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ configurable: false, choice: { every: '30m' } })] }); + expect(fixed.desired[0].recurrence).toEqual({ every: '1h' }); + expect(fixed.problems).toEqual([{ key: 'POLL', problems: ['This schedule is not configurable'] }]); + }); + + it('lets a configurable schedule be paused by the installation, and only then', () => { + expect(desiredSchedules({ integrationName: 'p', integrationId: 1, entries: [entry({ choice: { enabled: false } })] }).desired[0].enabled).toBe(false); + expect(desiredSchedules({ integrationName: 'p', integrationId: 1, entries: [entry({ configurable: false, choice: { enabled: false } })] }).desired[0].enabled).toBe(true); + }); + + it('builds host-safe names within 64 characters that stay unique', () => { + expect(scheduleJobId('hubspot', 42, 'ONGOING_SYNC')).toBe('hubspot-42-ongoing_sync'); + const long = scheduleJobId('data-defined', '65f1c0ffee0000000000abcd', 'SYNC_A_VERY_LONG_SYNC_IDENTIFIER_THAT_GOES_ON_ONGOING'); + const other = scheduleJobId('data-defined', '65f1c0ffee0000000000abcd', 'SYNC_A_VERY_LONG_SYNC_IDENTIFIER_THAT_GOES_ON_ONGOING_2'); + expect(long.length).toBeLessThanOrEqual(64); + expect(long).not.toBe(other); + expect(long).toMatch(/^[0-9A-Za-z_.-]+$/); + expect(scheduleJobId('a b', 'c/d', 'E:F')).toBe('a-b-c-d-e-f'); + }); +}); + +describe('IntegrationBase recurring schedules', () => { + let service, integration; + + class Poller extends IntegrationBase { + static Definition = { + name: 'poller', + version: '1.0.0', + modules: {}, + schedules: { + ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, + DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, + }, + }; + } + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + process.env.POLLER_QUEUE_URL = QUEUE_URL; + service = new MockSchedulerAdapter(); + integration = new Poller(); + integration.id = 'int-1'; + integration.status = 'IN_CREATION'; + integration.config = {}; + integration.schedulerCommands = createSchedulerCommands({ integrationName: 'poller', schedulerService: service }); + integration.updateIntegrationStatus = { execute: jest.fn().mockResolvedValue(true) }; + integration.updateIntegrationMessages = { execute: jest.fn().mockResolvedValue(true) }; + integration.patchIntegrationConfig = { execute: jest.fn(async (id, patch) => ({ config: (integration.config = { ...integration.config, ...patch }) })) }; + integration.getConfigOptions = jest.fn().mockResolvedValue({ jsonSchema: {}, uiSchema: {} }); + }); + afterEach(() => { jest.restoreAllMocks(); delete process.env.POLLER_QUEUE_URL; }); + + it('onCreate creates one schedule per declared entry for this installation and records them on config', async () => { + await integration.onCreate(); + const schedules = service._getSchedules(); + expect(Object.keys(schedules).sort()).toEqual(['poller-int-1-daily_digest', 'poller-int-1-ongoing_sync']); + expect(schedules['poller-int-1-daily_digest'].recurrence).toEqual({ cron: '0 8 * * 1-5', timezone: 'America/New_York' }); + expect(service._simulateTrigger('poller-int-1-ongoing_sync', { id: 'x', scheduledFor: 't' })).toMatchObject({ event: 'ONGOING_SYNC', data: { integrationId: 'int-1', scheduleKey: 'ONGOING_SYNC' } }); + expect(integration.config.runtime.schedules).toEqual({ + ONGOING_SYNC: expect.objectContaining({ jobId: 'poller-int-1-ongoing_sync', recurrence: { every: '1h' }, event: 'ONGOING_SYNC' }), + DAILY_DIGEST: expect.objectContaining({ jobId: 'poller-int-1-daily_digest' }), + }); + }); + + it('onUpdate applies the installation\'s own interval, and refuses one below the floor with a warning', async () => { + await integration.onCreate(); + await integration.onUpdate({ config: { schedules: { ONGOING_SYNC: { every: '20m' } } } }); + expect(service._getSchedules()['poller-int-1-ongoing_sync'].recurrence).toEqual({ every: '20m' }); + + await integration.onUpdate({ config: { schedules: { ONGOING_SYNC: { every: '1m' }, DAILY_DIGEST: { cron: '0 * * * *' } } } }); + expect(service._getSchedules()['poller-int-1-ongoing_sync'].recurrence).toEqual({ every: '1h' }); // back to the default + expect(service._getSchedules()['poller-int-1-daily_digest'].recurrence.cron).toBe('0 8 * * 1-5'); // fixed + const warnings = integration.updateIntegrationMessages.execute.mock.calls.map((c) => c[3]); + expect(warnings).toEqual([ + 'ONGOING_SYNC: Runs every 1 minute at its most frequent; the minimum is 15m', + 'DAILY_DIGEST: This schedule is not configurable', + ]); + }); + + it('onDelete removes every recorded schedule, including one the code no longer declares', async () => { + await integration.onCreate(); + integration.config.runtime.schedules.LEGACY = { jobId: 'poller-int-1-legacy', recurrence: { every: '1d' }, event: 'LEGACY' }; + await service.upsertRecurring({ scheduleName: 'poller-int-1-legacy', recurrence: { every: '1d' }, queueResourceId: 'arn:q', payload: {} }); + await integration.onDelete(); + expect(service._getSchedules()).toEqual({}); + expect(integration.config.runtime.schedules).toEqual({}); + }); + + it('a schedule the host refuses stays unrecorded and is logged; the lifecycle carries on', async () => { + jest.spyOn(service, 'upsertRecurring').mockRejectedValue(new Error('AccessDenied')); + const error = jest.spyOn(integration.logger, 'error').mockImplementation(() => {}); + await integration.onCreate(); + expect(integration.updateIntegrationStatus.execute).toHaveBeenCalledWith('int-1', 'ENABLED'); + expect(integration.config.runtime).toBeUndefined(); + expect(error).toHaveBeenCalledWith('Schedules not reconciled', expect.objectContaining({ eventName: expect.stringMatching(/schedules_failed$/) })); + }); + + it('does nothing at all for an integration with no schedules', async () => { + class Plain extends IntegrationBase { static Definition = { name: 'plain', version: '1.0.0', modules: {} }; } + const plain = new Plain(); + plain.id = 'p1'; + plain.config = {}; + plain.patchIntegrationConfig = { execute: jest.fn() }; + expect(await plain.syncSchedules()).toBeNull(); + expect(await plain.syncSchedules({ remove: true })).toBeNull(); + expect(plain.patchIntegrationConfig.execute).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js b/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js index daaf5c03b..cd31e4aa8 100644 --- a/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js +++ b/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js @@ -8,8 +8,9 @@ * - Creating IAM Role for EventBridge Scheduler to send messages to SQS * - Adding necessary IAM statements for Lambda to create/delete schedules * - * This builder enables integrations to schedule one-time jobs (e.g., webhook renewals) - * using AWS EventBridge Scheduler. + * This builder enables integrations to schedule one-time jobs (e.g., webhook renewals) and + * recurring per-installation jobs (Definition.schedules, data-defined schedule triggers) + * using AWS EventBridge Scheduler. Recurring jobs are upserted, hence UpdateSchedule. */ const { InfrastructureBuilder, ValidationResult } = require('../shared/base-builder'); @@ -173,6 +174,7 @@ class SchedulerBuilder extends InfrastructureBuilder { Effect: 'Allow', Action: [ 'scheduler:CreateSchedule', + 'scheduler:UpdateSchedule', 'scheduler:DeleteSchedule', 'scheduler:GetSchedule', ], From 5ba8cfaf0787d59472239914be0e5a2e4dd53c69 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:13:28 +0000 Subject: [PATCH 2/3] docs(adr): ADR-063 recurring schedules follow ADR-055; versioning row is built here - Section 15a: ADR-055 governs recurring work (Definition.triggers, one tick, due check); the per-installation host schedule is being reworked. - Section 17: the EventBridge row no longer says registration is not built. - Do not name a specific integration's crons. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NGC9SnFuqdjX2xH2qeu8vw --- .../063-integration-definitions-as-data.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/architecture-decisions/063-integration-definitions-as-data.md b/docs/architecture-decisions/063-integration-definitions-as-data.md index c649a5f44..82d5fa2dc 100644 --- a/docs/architecture-decisions/063-integration-definitions-as-data.md +++ b/docs/architecture-decisions/063-integration-definitions-as-data.md @@ -516,7 +516,7 @@ had no `runtime`, so a query reading an install result silently got nothing. Schedule-triggered workflows and polling syncs run on a host schedule of their own, one per installation. Recurring jobs are not specific to data-defined integrations, so the mechanism is in core and a hand-written integration uses -the same one (Quo's crons, for example, which are set by hand per integration today): +the same one (the crons that some integrations set by hand today, for example): ```js static Definition = { @@ -549,6 +549,8 @@ tick runs once, because `tick::` is the workflow's dedupe i major change. An upgrade keeps a choice only where the new version still allows it, and otherwise reports a `reschedule` action. +**This design is being reworked to follow ADR-055.** The review of this change requires [ADR-055](./055-recurring-triggers.md) to govern recurring work. ADR-055 declares recurrence as `Definition.triggers`, provisions one schedule per integration, and has the framework fan out at each tick to the `ENABLED` installations that are due. No per-installation schedule exists. This section does the opposite on that last point: one host schedule per installation, reconciled by `syncSchedules()`. The rework changes the declaration (`Definition.triggers`) and replaces the per-installation host schedules with a tick and a due check. The Port, Adapters, Commands and Lifecycle rows above describe what this change builds, not the target, and the schedule names and `config.runtime.schedules` record may change with it. The code change is a separate task. A definition's `schedule` triggers and polling syncs reach the same mechanism, so they follow ADR-055 as well. + Ticks for an installation that is not `ENABLED` are discarded by the queue worker, as one-time jobs already are. Pausing the host schedule when the status changes is not done yet. Changing status does not go through `onUpdate`, so it needs a hook of its own. @@ -605,7 +607,7 @@ code-defined classes; this is the answer only where the class is the shared inte |---|---| | Lambda module-scope caching | Nothing version-specific is read at module scope. The definition is fetched when an installation is hydrated for an event and cached by `id@version`, so one warm Lambda serves several versions at once and a publish needs no deploy | | In-flight SQS jobs | Every message carries `definitionVersion`, and `dispatch` runs it on that version: ADR-030's "dispatch by the payload's declared version" | -| EventBridge Scheduler entries | The rule: a scheduled run carries no version and runs on the installation's current pin, because a schedule belongs to the installation, not to the version that created it. `compile` produces each installation's `schedules`, so an upgrade that changes a cron yields a different list. **Registering that list is not built in v0.1** (§19), so this row is the contract for when it is | +| EventBridge Scheduler entries | The rule: a scheduled run carries no version and runs on the installation's current pin, because a schedule belongs to the installation, not to the version that created it. `compile` produces each installation's `schedules`, so an upgrade that changes a cron yields a different list. **Registering that list is built here** (§15a): one host schedule per installation, reconciled on create, update and delete. This row is how an upgrade interacts with it | | Signed webhook URLs | URLs are per installation, not per version, so an upgrade re-registers nothing. Only a change to `lifecycle.onInstall` re-runs install, and `planUpgrade` lists that as a `reinstall` action | | Rollback | Moving the pin back is a config write (ADR-030's Path C behaviour). An identity change still needs links rebuilt in both directions, and `planUpgrade` says so | | Global routing / edge | Unchanged: routing is by installation id, and the version is resolved at the origin from the record | From d7b4f7f0cfd519e0a8df8e9ef0fa451cc313643a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 21:36:35 +0000 Subject: [PATCH 3/3] feat(core)!: recurring triggers per ADR-055 replace per-installation schedules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-055 governs: no schedule exists per installation. The per-installation EventBridge mechanism from the previous commit (IntegrationBase.syncSchedules, reconcileRecurringJobs, upsertRecurring on the scheduler port, and config.runtime.schedules) is removed, and lifecycle hooks no longer touch the scheduler. - Definition.triggers (ADR-055 §1) replaces Definition.schedules: every (constant or a function of config), at + timezone, tick, when, overlap and a handler (function or method name). Amendment: configurable: true | { min } lets an installation choose its timing at config.triggers., and a five-field cron + timezone is a calendar trigger that may differ per installation, due when it fired in (previous tick, this tick]. - integrations/triggers.js: validation, provisioningOf, resolveTrigger and the stateless due check (every: slot mod n == hash(id) mod n, a chosen interval rounded up to whole ticks). recurrence.js gains lastFiringIn (time-zone aware); validateRecurrence and the configurable floor are unchanged. - Tick: RunTriggerTick pages ENABLED installations 500 at a time (findIntegrations gains limit/after on PostgreSQL, MongoDB and DocumentDB), sends { event: 'FRIGG_TRIGGER_RUN', data: { integrationId, trigger, scheduledFor } } with batchSend and DelaySeconds jitter, and hands its cursor to the queue as FRIGG_TRIGGER_TICK near the deadline. Handler: handlers/workers/integration-triggers.js. - Run: IntegrationBase.onTriggerRun via the RunTrigger use case. Not ENABLED is discarded; overlap 'skip' leases on one TRIGGER: Process record per installation; a failed run is marked ERROR and rethrown. - Data-defined: compile emits the same trigger entries (compiled.triggers); the class declares instanceTriggers: { tick: '5m' } and resolves each installation's triggers from its pinned definition and config at the tick. Configurable timing, floors, the form, versioning (dropping a disallowed choice on upgrade) and the tick:: dedupe keep working. - MongoDB findIntegrations({ type }) used a JSON path filter the Prisma MongoDB connector refuses; it now reads matching ids with findRaw. - devtools: TriggerBuilder (runs when any integration declares triggers) emits {integration}Triggers with one rate() schedule per distinct tick (its firing time as input) and one method: scheduler event per at trigger; the framework adds the invoke permissions, and the tick needs no runtime scheduler IAM. getIntegrationFunctionNames includes the function so a scoped queue URL reaches it. SchedulerBuilder is back to one-time jobs. - Docs: ADR-063 §15a, CLAUDE.md and INTEGRATION-PATTERNS.md describe triggers instead of per-installation schedules and CRON events. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NGC9SnFuqdjX2xH2qeu8vw --- CLAUDE.md | 36 ++- .../063-integration-definitions-as-data.md | 74 ++--- docs/guides/INTEGRATION-PATTERNS.md | 22 +- .../commands/scheduler-commands.js | 95 +----- packages/core/database/documentdb-utils.js | 2 + .../handlers/workers/integration-triggers.js | 25 ++ .../workers/integration-triggers.test.js | 28 ++ .../scheduler/__tests__/recurrence.test.js | 88 ++++++ .../__tests__/recurring-schedules.test.js | 228 --------------- .../eventbridge-scheduler-adapter.js | 70 +---- .../core/infrastructure/scheduler/index.js | 9 +- .../scheduler/mock-scheduler-adapter.js | 33 +-- .../infrastructure/scheduler/recurrence.js | 45 ++- .../scheduler/scheduler-service-interface.js | 36 +-- .../core/integrations/data-defined/README.md | 16 +- .../core/integrations/data-defined/compile.js | 13 +- .../data-defined/data-defined-integration.js | 68 +++-- .../schema/installation.v0.1.schema.json | 41 +-- .../data-defined/tests/data-defined.test.js | 2 +- .../data-defined/tests/schedules.test.js | 151 ++++++---- .../integrations/data-defined/versioning.js | 2 +- .../core/integrations/integration-base.js | 124 ++++---- .../integration-repository-documentdb.js | 8 +- .../integration-repository-interface.js | 3 + .../integration-repository-mongo.js | 23 +- .../integration-repository-paging.test.js | 112 ++++++++ .../integration-repository-postgres.js | 6 +- packages/core/integrations/schedules.js | 89 ------ .../tests/integration-base-schedules.test.js | 137 --------- .../tests/trigger-runtime.test.js | 271 ++++++++++++++++++ .../core/integrations/tests/triggers.test.js | 169 +++++++++++ packages/core/integrations/triggers.js | 244 ++++++++++++++++ .../use-cases/run-trigger-tick.js | 132 +++++++++ .../integrations/use-cases/run-trigger.js | 77 +++++ packages/core/queues/queuer-util.js | 11 +- packages/core/queues/queuer-util.test.js | 13 + .../domains/scheduler/scheduler-builder.js | 6 +- .../domains/scheduler/trigger-builder.js | 114 ++++++++ .../domains/scheduler/trigger-builder.test.js | 82 ++++++ .../domains/shared/function-environments.js | 10 +- .../shared/function-environments.test.js | 14 + .../infrastructure/infrastructure-composer.js | 2 + 42 files changed, 1788 insertions(+), 943 deletions(-) create mode 100644 packages/core/handlers/workers/integration-triggers.js create mode 100644 packages/core/handlers/workers/integration-triggers.test.js create mode 100644 packages/core/infrastructure/scheduler/__tests__/recurrence.test.js delete mode 100644 packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js create mode 100644 packages/core/integrations/repositories/integration-repository-paging.test.js delete mode 100644 packages/core/integrations/schedules.js delete mode 100644 packages/core/integrations/tests/integration-base-schedules.test.js create mode 100644 packages/core/integrations/tests/trigger-runtime.test.js create mode 100644 packages/core/integrations/tests/triggers.test.js create mode 100644 packages/core/integrations/triggers.js create mode 100644 packages/core/integrations/use-cases/run-trigger-tick.js create mode 100644 packages/core/integrations/use-cases/run-trigger.js create mode 100644 packages/devtools/infrastructure/domains/scheduler/trigger-builder.js create mode 100644 packages/devtools/infrastructure/domains/scheduler/trigger-builder.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 74dd56661..005227d1b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -180,7 +180,7 @@ Key patterns covered: - **Process Model**: Track long-running operations with state management (`INITIALIZING` → `PROCESSING` → `COMPLETED`) - **friggCommands**: Standardized interface for persisting integration config (`createFriggCommands()`) - **QueueManager**: AWS SQS wrapper for async job processing with rate limiting and fan-out -- **Integration Events**: Define `USER_ACTION`, `CRON`, `QUEUE`, and `WEBHOOK` event handlers +- **Integration Events**: Define `USER_ACTION`, `QUEUE`, and `WEBHOOK` event handlers; recurring work is `Definition.triggers` - **SyncOrchestrator**: Coordinate sync operations across entity types Quick example: @@ -189,13 +189,17 @@ Quick example: const { createFriggCommands } = require('@friggframework/core'); class MyIntegration extends IntegrationBase { + static Definition = { + // ...name, version, modules + triggers: { ONGOING_SYNC: { every: '15m', handler: 'deltaSync' } }, // recurring (ADR-055) + }; + constructor(params) { super(params); this.commands = createFriggCommands({ integrationClass: MyIntegration }); this.events = { INITIAL_SYNC: { type: 'USER_ACTION', handler: this.startSync.bind(this) }, - ONGOING_SYNC: { type: 'CRON', handler: this.deltaSync.bind(this) }, PROCESS_BATCH: { handler: this.processBatch.bind(this) } }; } @@ -917,7 +921,7 @@ When working on the Frigg Framework, always prioritize finding the **best soluti ### Scheduler Commands -The Frigg framework provides scheduler commands for one-time and recurring jobs using AWS EventBridge Scheduler: +The Frigg framework provides scheduler commands for scheduling one-time jobs using AWS EventBridge Scheduler: ```javascript const { createSchedulerCommands } = require('@friggframework/core'); @@ -942,26 +946,32 @@ await schedulerCommands.deleteJob('zoho-notif-renewal-abc123'); const status = await schedulerCommands.getJobStatus('zoho-notif-renewal-abc123'); ``` -**Recurring schedules, one per installation** (`Definition.schedules`): declare them on the integration and -`IntegrationBase` creates, retimes and deletes them in `onCreate` / `onUpdate` / `onDelete` (an override calls -`this.syncSchedules()` itself). Recurrence is host-neutral; each scheduler adapter translates it. +**Recurring work uses triggers, not per-installation schedules** (`Definition.triggers`, ADR-055). Devtools +provisions one `{integration}Triggers` function per integration that declares any, with one schedule per distinct +tick. Each tick pages through the ENABLED installations and queues `{ event: 'FRIGG_TRIGGER_RUN', data: +{ integrationId, trigger, scheduledFor } }` for each one that is due; the queue worker runs it through +`IntegrationBase.onTriggerRun`. Lifecycle hooks never touch the scheduler: a disabled or deleted installation is +simply not due, and a config change applies at the next tick. ```javascript static Definition = { name: 'housecallpro', - schedules: { - ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // user may set config.schedules.ONGOING_SYNC - DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed + triggers: { + ONGOING_SYNC: { every: '1h', configurable: { min: '15m' }, handler: 'deltaSync' }, // user may set config.triggers.ONGOING_SYNC + DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York', handler: digest }, // due when it fired since the last tick + NIGHTLY: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler: 'reconcile' }, // own schedule, same for everyone }, }; ``` -The host's state is recorded at `config.runtime.schedules`. Each firing sends `{ event, data: { integrationId, -scheduleKey, tick: { id, scheduledFor } } }` to the integration's queue, so a handler can dedupe a redelivered tick. -For one job without the lifecycle, use `schedulerCommands.upsertRecurringJob({ jobId, recurrence, event, payload, queueUrl })`. +Handlers are called as `handler({ trigger, scheduledFor }, { modules, config, log, integration })` (a string names a +method on the instance). `every` is due when `slot mod n == hash(integrationId) mod n`, so installations spread over +the cadence; a chosen interval rounds up to whole ticks. `overlap: 'skip'` (default) keeps a `TRIGGER:` Process +record per installation as the lease. Pure rules: `integrations/triggers.js`; tick: `integrations/use-cases/run-trigger-tick.js`. +`createSchedulerCommands` stays the API for one-time jobs. **Key Features**: -- Uses AWS EventBridge Scheduler for reliable one-time and recurring job execution +- Uses AWS EventBridge Scheduler for reliable one-time job execution - Targets SQS queues (accepts `queueUrl`, derives ARN internally) - Mock scheduler available for local development (`SCHEDULER_PROVIDER=mock`) - Auto-cleanup with dead letter queue support diff --git a/docs/architecture-decisions/063-integration-definitions-as-data.md b/docs/architecture-decisions/063-integration-definitions-as-data.md index c649a5f44..de725cc52 100644 --- a/docs/architecture-decisions/063-integration-definitions-as-data.md +++ b/docs/architecture-decisions/063-integration-definitions-as-data.md @@ -125,7 +125,7 @@ Because a definition can only compose allow-listed, versioned code, a workspace- ## Implementation (v0.1) The decision above was written before any code existed. v0.1 is a working implementation in -`packages/core/integrations/data-defined/`, with 136 tests (the persistence suite runs against real PostgreSQL and +`packages/core/integrations/data-defined/`, with 137 tests (the persistence suite runs against real PostgreSQL and MongoDB). It runs 16 example definitions (Quo ⇄ HubSpot is the reference: seven workflows, one two-way sync, two UI surfaces) against fake API modules, and checks 21 capability manifests. This section records how it works, where it changed the sketch above, and what it does not do yet. File @@ -369,10 +369,10 @@ tables the runtime needs: | query `recent-activity` | `Q_RECENT_ACTIVITY` | `ROUTE` | | UI surface `quo-card` | `UI_QUO_CARD`, rendered by `POST /api/integrations/:id/actions/UI_QUO_CARD` | `ROUTE` | -The other outputs are `webhooks` (module → topic → event names; a pattern may end in `.*`), `schedules` and +The other outputs are `webhooks` (module → topic → event names; a pattern may end in `.*`), `triggers` (§15a) and `userActions`. -**Only enabled items are compiled.** A workflow the customer has off has no event, no route and no schedule. It +**Only enabled items are compiled.** A workflow the customer has off has no event, no route and no trigger. It does not exist at runtime, so "disabled but still firing" cannot happen. ### 13. Two deployment shapes @@ -462,7 +462,9 @@ whose account is its own. step 3 again. Idempotent capabilities (upserts, `links.find` before create) are the v0.1 answer. - Per-step checkpointing comes after v0.1. -**A schedule fires.** `CRON` → `WF_STALE_CONTACTS`, with no event id, so there is no claim. Steps such as +**A schedule fires.** The integration's trigger tick finds the installation due and queues `FRIGG_TRIGGER_RUN`, +which runs `WF_STALE_CONTACTS` with the event id `tick:WF_STALE_CONTACTS:`, so a redelivered run is +claimed once (§15a). Steps such as `hubspot.contact.search` followed by `foreach` run in the same interpreter. **A user clicks a button** in a UI surface. `USER_ACTION` → `WF_TEXT_FROM_CARD`, with the surface's record as @@ -503,55 +505,61 @@ genuine (HubSpot signs its UI-extension requests) is the renderer's job, not Fri | Moment | What runs | Code | |---|---|---| | **Install** | The customer connects both modules (the existing auth flow), picks settings, and the record is created with the pin to the latest `PUBLISHED` version. `onCreate` loads the definition and runs `lifecycle.onInstall`, for example `quo.webhook.create` pointed at this installation's webhook URL. Every install step with an `id` has its result saved to `config.runtime.installed.` with `patchConfig` | `onCreate` | -| **Configure** | The customer toggles workflows, changes settings, overrides a mapping or, where the definition allows it, retimes a schedule. `onUpdate` merges config, then recompiles: enabling a workflow adds its event, routes and host schedule, and disabling removes them (§15a). No deploy, and no other installation is affected | `onUpdate` | +| **Configure** | The customer toggles workflows, changes settings, overrides a mapping or, where the definition allows it, retimes a schedule. `onUpdate` merges config, then recompiles: enabling a workflow adds its event and routes, and disabling removes them. Its trigger needs nothing: the next tick reads the new config (§15a). No deploy, and no other installation is affected | `onUpdate` | | **Update** (new version) | Nothing happens to running installations. Each stays on its pin until it is moved, as in §17 | `planUpgrade` | -| **Uninstall** | `onDelete` runs `lifecycle.onUninstall`, which reads what install created (`quo.webhook.delete` with `$.install.runtime.installed.hook.id`; a `foreach` over several webhook ids for Shopify), deletes the installation's host schedules, then the framework removes the record | `onDelete` | +| **Uninstall** | `onDelete` runs `lifecycle.onUninstall`, which reads what install created (`quo.webhook.delete` with `$.install.runtime.installed.hook.id`; a `foreach` over several webhook ids for Shopify), then the framework removes the record. There is no schedule to delete: a deleted installation is simply not due at the next tick | `onDelete` | `$.install` is built by one function, `installContext`, everywhere a definition runs: workflows, queries, UI surfaces, lifecycle steps and option lists. It holds the integration settings with their defaults, `webhookUrl`, `integrationId`, `accounts` and `runtime.installed`. The first cut built it four ways, and the query and UI versions had no `runtime`, so a query reading an install result silently got nothing. -### 15a. Recurring schedules, one per installation +### 15a. Recurring triggers (ADR-055), with timing an installation may choose -Schedule-triggered workflows and polling syncs run on a host schedule of their own, one per installation. Recurring -jobs are not specific to data-defined integrations, so the mechanism is in core and a hand-written integration uses -the same one (Quo's crons, for example, which are set by hand per integration today): +Schedule-triggered workflows and polling syncs are ADR-055 recurring triggers. Nothing is scheduled per +installation. Each integration with triggers gets one function, `{integration}Triggers`, with one schedule per +distinct tick (and one per `at` trigger); at each tick it pages through the ENABLED installations and queues +`{ event: 'FRIGG_TRIGGER_RUN', data: { integrationId, trigger, scheduledFor } }` for each one that is due. A disabled +or deleted installation is not due, a config change or an upgrade applies at the next tick, and there is nothing for +ADR-032 to clean up. Recurring jobs are not specific to data-defined integrations, so the mechanism is in core and a +hand-written integration declares the same thing: ```js static Definition = { name: 'housecallpro', - schedules: { - ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // a user may choose >= 15m - DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed + triggers: { + ONGOING_SYNC: { every: '1h', configurable: { min: '15m' }, handler: 'deltaSync' }, // a user may choose >= 15m + DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York', configurable: true, handler: 'digest' }, + NIGHTLY: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler: 'reconcile' }, // one time for everyone }, }; ``` -A data-defined definition says the same thing on the trigger (`{ "type": "schedule", "cron" | "every", "timezone", -"configurable" }`) or on a polling sync (`ongoing: { "mode": "poll", "every", "configurable" }`). The pieces: +A data-defined definition says it on the trigger (`{ "type": "schedule", "cron" | "every", "timezone", "configurable" }`) +or on a polling sync (`ongoing: { "mode": "poll", "every", "configurable" }`). `compile` turns both into the same +trigger entries (`compiled.triggers`), and the data-defined class resolves them per installation: it declares +`Definition.instanceTriggers: { tick: '5m' }`, and its static `triggersFor({ config })` compiles the installation's +pinned definition with its config at each tick. Each entry's handler runs its compiled event. | Layer | What | File | |---|---|---| -| Recurrence | Host-neutral: `{ every: '15m' }` or a five-field `{ cron, timezone }`. `validateRecurrence` refuses what no host can run: both day fields restricted, a cron that never fires, an unknown zone. `minGapMinutes` measures what an expression really does, so a floor of 15m refuses `0,5 * * * *` | `infrastructure/scheduler/recurrence.js` | -| Port | `upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled })` beside the one-time methods. Each adapter translates the recurrence. The payload may carry `TICK.id` / `TICK.scheduledFor`, which the host fills per firing | `scheduler-service-interface.js` | -| Adapters | EventBridge: `rate(...)` or a six-field `cron(... ? ...)` with weekdays written as names, because EventBridge numbers them 1-7 from Sunday. It updates first, creates when the schedule is missing, and updates again after losing a create race. The tick is filled from ``. Mock: in memory, with `_simulateTrigger(name, tick)` | `eventbridge-scheduler-adapter.js`, `mock-scheduler-adapter.js` | -| Commands | `upsertRecurringJob` and `reconcileRecurringJobs({ desired, stored, queueUrl })`. A job is written only when its recurrence, event, payload or state changed, and deleted when it is no longer wanted. A write the host refuses leaves the stored entry as it was, so the next reconcile retries it | `application/commands/scheduler-commands.js` | -| Desired set | The definition's entries plus the installation's choices. A choice is honoured only where the definition allows it and within its floor. An invalid choice keeps the default and adds a warning message to the integration, so a bad edit never stops a schedule | `integrations/schedules.js` | -| Lifecycle | `IntegrationBase.syncSchedules()` runs from the default `onCreate`, `onUpdate` and `onDelete`. It records what the host holds at `config.runtime.schedules.`. An integration with no schedules does nothing. The data-defined class supplies its entries by compiling the current config, and also calls it after an upgrade | `integration-base.js` | - -The installation's choice lives in its config: `config.schedules.` for a hand-written integration, and +| Recurrence | Host-neutral: `{ every: '15m' }` or a five-field `{ cron, timezone }`. `validateRecurrence` refuses both day fields restricted, a cron that never fires, an unknown zone. `minGapMinutes` measures what an expression really does, so a floor of 15m refuses `0,5 * * * *`. `lastFiringIn(cron, from, to)` answers "did it fire since the last tick" in the cron's time zone | `infrastructure/scheduler/recurrence.js` | +| Declaration and due check | `validateTriggers`, `provisioningOf` (the ticks and `at` triggers devtools provisions), `resolveTrigger` (config, `when`, the installation's choice within the floor, the cadence in ticks) and `dueRun`. An `every` trigger is due when `slot mod n == hash(integrationId) mod n`; a `cron` trigger when it fired in (previous tick, this tick]; an `at` trigger whenever its own schedule runs | `integrations/triggers.js` | +| Tick | Pages ENABLED installations 500 at a time by id (`findIntegrations({ type, status, limit, after })`), sends due runs with `batchSend` and per-run `DelaySeconds` jitter, and hands its cursor to the queue (`FRIGG_TRIGGER_TICK`) when it nears the Lambda deadline | `integrations/use-cases/run-trigger-tick.js`, `handlers/workers/integration-triggers.js` | +| Run | The queue worker hydrates by `integrationId` and dispatches `FRIGG_TRIGGER_RUN` to `IntegrationBase.onTriggerRun`. Not ENABLED: discarded. `overlap: 'skip'`: a Process record `TRIGGER:` per installation is the lease, and a run still going (touched within 2 x tick) makes the next one skip | `integrations/use-cases/run-trigger.js` | +| Provisioning | `TriggerBuilder` (runs when any integration declares triggers): the function, a `rate()` schedule event per tick with its firing time as input, and a `method: scheduler` event per `at` trigger. No runtime scheduler IAM | `devtools/.../scheduler/trigger-builder.js` | + +The installation's choice lives in its config: `config.triggers.` for a hand-written integration, and `workflows..schedule` / `syncs..schedule` for a data-defined one, offered in the configure form only where -the definition allows it. Host schedule names are `--`, kept within -EventBridge's 64 characters. Each firing sends `{ event, data: { integrationId, scheduleKey, tick } }` to the -integration's queue, so the existing queue worker hydrates the installation and dispatches the event. A redelivered -tick runs once, because `tick::` is the workflow's dedupe id. Taking configurability away is a -major change. An upgrade keeps a choice only where the new version still allows it, and otherwise reports a -`reschedule` action. +the definition allows it. A choice the definition does not allow, or one under the floor, keeps the default timing +and `onUpdate` adds a warning message saying why. An interval runs on whole multiples of the tick (a 20-minute +choice on a 15-minute tick runs every 30 minutes), and a cron runs on the first tick after it fires, so its delay is +at most one tick. A redelivered run, or two ticks in one slot, run once: the slot decides `scheduledFor`, and +`tick::` is the workflow's dedupe id. Taking configurability away is a major change. An upgrade +keeps a choice only where the new version still allows it, and otherwise reports a `reschedule` action. -Ticks for an installation that is not `ENABLED` are discarded by the queue worker, as one-time jobs already are. -Pausing the host schedule when the status changes is not done yet. Changing status does not go through `onUpdate`, -so it needs a hook of its own. +The two additions to ADR-055 (`configurable` floors, and `cron` calendar triggers that differ per installation) are +proposed as its "Amendment (ADR-063)". ### 16. UI surfaces @@ -605,7 +613,7 @@ code-defined classes; this is the answer only where the class is the shared inte |---|---| | Lambda module-scope caching | Nothing version-specific is read at module scope. The definition is fetched when an installation is hydrated for an event and cached by `id@version`, so one warm Lambda serves several versions at once and a publish needs no deploy | | In-flight SQS jobs | Every message carries `definitionVersion`, and `dispatch` runs it on that version: ADR-030's "dispatch by the payload's declared version" | -| EventBridge Scheduler entries | The rule: a scheduled run carries no version and runs on the installation's current pin, because a schedule belongs to the installation, not to the version that created it. `compile` produces each installation's `schedules`, so an upgrade that changes a cron yields a different list. **Registering that list is not built in v0.1** (§19), so this row is the contract for when it is | +| EventBridge Scheduler entries | There are none per installation (§15a). A trigger run carries no version and runs on the installation's current pin, because the tick compiles the pinned definition each time it runs. An upgrade that changes a cron changes what the next tick finds due, and registers nothing | | Signed webhook URLs | URLs are per installation, not per version, so an upgrade re-registers nothing. Only a change to `lifecycle.onInstall` re-runs install, and `planUpgrade` lists that as a `reinstall` action | | Rollback | Moving the pin back is a config write (ADR-030's Path C behaviour). An identity change still needs links rebuilt in both directions, and `planUpgrade` says so | | Global routing / edge | Unchanged: routing is by installation id, and the version is resolved at the origin from the record | diff --git a/docs/guides/INTEGRATION-PATTERNS.md b/docs/guides/INTEGRATION-PATTERNS.md index eddba15fa..e63eff0d6 100644 --- a/docs/guides/INTEGRATION-PATTERNS.md +++ b/docs/guides/INTEGRATION-PATTERNS.md @@ -324,7 +324,7 @@ Define event handlers in your integration class to handle different types of ope | Type | Purpose | Trigger | |------|---------|---------| | `USER_ACTION` | User-initiated operations | UI button click | -| `CRON` | Scheduled operations | CloudWatch Events | +| Trigger (`Definition.triggers`) | Recurring operations | The integration's trigger tick (ADR-055) | | `QUEUE` | Queue-triggered handlers | SQS messages | | `WEBHOOK` | External webhook events | HTTP POST from external service | @@ -335,6 +335,11 @@ class MyIntegration extends IntegrationBase { static Definition = { name: 'my-integration', version: '1.0.0', + // Recurring work is a trigger, not a CRON event: one tick per integration fans out to + // the ENABLED installations that are due (ADR-055). The handler is a method name. + triggers: { + ONGOING_SYNC: { every: '15m', handler: 'startOngoingSync' }, + }, // ... other definition properties }; @@ -350,13 +355,6 @@ class MyIntegration extends IntegrationBase { description: 'Sync all records from source to destination' }, - // Cron-triggered ongoing sync - ONGOING_SYNC: { - type: 'CRON', - handler: this.startOngoingSync.bind(this), - schedule: 'rate(15 minutes)' - }, - // Queue handlers FETCH_PAGE: { type: 'QUEUE', @@ -678,6 +676,10 @@ class MyIntegration extends IntegrationBase { destination: { definition: DestinationApiDefinition } }, + triggers: { + ONGOING_SYNC: { every: '15m', handler: 'startOngoingSync' } + }, + events: [ 'SYNC_STARTED', 'SYNC_COMPLETED', @@ -715,10 +717,6 @@ class MyIntegration extends IntegrationBase { title: 'Initial Sync', description: 'Sync all data from source to destination' }, - ONGOING_SYNC: { - type: 'CRON', - handler: this.startOngoingSync.bind(this) - }, FETCH_PAGE: { handler: this.fetchPageHandler.bind(this) }, diff --git a/packages/core/application/commands/scheduler-commands.js b/packages/core/application/commands/scheduler-commands.js index 5ce464083..006cf82e7 100644 --- a/packages/core/application/commands/scheduler-commands.js +++ b/packages/core/application/commands/scheduler-commands.js @@ -19,7 +19,7 @@ * }); */ -const { createSchedulerService, TICK, validateRecurrence, recurrenceKey } = require('../../infrastructure/scheduler'); +const { createSchedulerService } = require('../../infrastructure/scheduler'); /** * Derive SQS ARN from SQS URL @@ -89,7 +89,7 @@ function createSchedulerCommands({ integrationName, schedulerService }) { return _schedulerService; } - const commands = { + return { /** * Schedule a one-time job to be executed at a specific time * @@ -179,96 +179,6 @@ function createSchedulerCommands({ integrationName, schedulerService }) { } }, - /** - * Create or update a recurring job. Every firing sends `{ event, data: { ...payload, tick } }` to the queue, - * where `tick` is `{ id, scheduledFor }` for that firing (when the host supplies them), so a handler can - * dedupe a redelivered tick. - * - * @param {Object} params - * @param {string} params.jobId - Unique, stable name for the schedule - * @param {{every?: string, cron?: string, timezone?: string}} params.recurrence - Host-neutral recurrence - * @param {string} params.event - Event name to trigger - * @param {Object} [params.payload] - Data sent on every firing - * @param {string} params.queueUrl - Target SQS queue URL - * @param {boolean} [params.enabled=true] - * @returns {Promise<{jobId, jobArn, created} | {jobId, warning} | {error, reason, code}>} - */ - async upsertRecurringJob({ jobId, recurrence, event, payload, queueUrl, enabled = true }) { - try { - const invalid = (message) => Object.assign(new Error(message), { code: 'INVALID_JOB_DATA' }); - if (!jobId) throw invalid('jobId is required'); - if (!event) throw invalid('event is required'); - if (!queueUrl) throw invalid('queueUrl is required'); - const problems = validateRecurrence(recurrence); - if (problems.length) throw invalid(`Invalid recurrence: ${problems.join('; ')}`); - - const service = getSchedulerService(); - if (!service) { - console.warn(`[${integrationName}] Scheduler not configured, skipping recurring job ${jobId}`); - return { jobId, jobArn: null, warning: 'Scheduler not configured' }; - } - const result = await service.upsertRecurring({ - scheduleName: jobId, - recurrence, - queueResourceId: deriveArnFromQueueUrl(queueUrl), - payload: { - event, - integrationName, - scheduleName: jobId, - data: { ...(payload || {}), tick: { id: TICK.id, scheduledFor: TICK.scheduledFor } }, - }, - enabled, - }); - return { jobId, jobArn: result.scheduledJobId, created: result.created }; - } catch (error) { - console.error(`[${integrationName}] Failed to upsert recurring job ${jobId}:`, error.message); - return mapErrorToResponse(error); - } - }, - - /** - * Make the host's recurring jobs match what an installation should have, and return what to store. - * - * `desired` is the full list for one installation; `stored` is what the last reconcile returned (kept on the - * integration config). A job is upserted when it is new or its recurrence, event, payload or state changed, - * and deleted when it is no longer desired. A failed upsert or delete leaves its stored entry as it was, so - * the next reconcile retries it; nothing is reported as scheduled that the host did not accept. - * - * @param {Object} params - * @param {Array<{key, jobId, recurrence, event, payload?, enabled?}>} params.desired - * @param {Object} [params.stored] - * @param {string} params.queueUrl - * @returns {Promise<{stored, created: string[], updated: string[], deleted: string[], unchanged: string[], failed: Array<{key, reason}>, warning?: string}>} - */ - async reconcileRecurringJobs({ desired = [], stored = {}, queueUrl }) { - const out = { stored: { ...(stored || {}) }, created: [], updated: [], deleted: [], unchanged: [], failed: [] }; - if (!getSchedulerService()) return { ...out, warning: 'Scheduler not configured' }; - const fingerprint = (d) => JSON.stringify([recurrenceKey(d.recurrence), d.event, d.payload || {}, d.enabled !== false, d.jobId]); - const wanted = new Set(); - for (const d of desired) { - wanted.add(d.key); - const prev = out.stored[d.key]; - const print = fingerprint(d); - if (prev && prev.fingerprint === print) { out.unchanged.push(d.key); continue; } - const res = await commands.upsertRecurringJob({ jobId: d.jobId, recurrence: d.recurrence, event: d.event, payload: d.payload, queueUrl, enabled: d.enabled !== false }); - if (res.error || res.warning) { out.failed.push({ key: d.key, reason: res.reason || res.warning }); continue; } - if (prev && prev.jobId !== d.jobId) { - const del = await commands.deleteJob(prev.jobId); - if (del.error) out.failed.push({ key: d.key, reason: `Old schedule ${prev.jobId} not deleted: ${del.reason}` }); - } - out.stored[d.key] = { jobId: d.jobId, recurrence: d.recurrence, event: d.event, enabled: d.enabled !== false, jobArn: res.jobArn, fingerprint: print, updatedAt: new Date().toISOString() }; - (prev ? out.updated : out.created).push(d.key); - } - for (const [key, prev] of Object.entries(out.stored)) { - if (wanted.has(key)) continue; - const del = await commands.deleteJob(prev.jobId); - if (del.error) { out.failed.push({ key, reason: del.reason }); continue; } - delete out.stored[key]; - out.deleted.push(key); - } - return out; - }, - /** * Delete a scheduled job * @@ -346,7 +256,6 @@ function createSchedulerCommands({ integrationName, schedulerService }) { } }, }; - return commands; } module.exports = { diff --git a/packages/core/database/documentdb-utils.js b/packages/core/database/documentdb-utils.js index b480377b0..afc4c6761 100644 --- a/packages/core/database/documentdb-utils.js +++ b/packages/core/database/documentdb-utils.js @@ -25,6 +25,8 @@ async function findMany(client, collection, filter = {}, options = {}) { if (options.projection) command.projection = options.projection; if (options.sort) command.sort = options.sort; if (options.limit) command.limit = options.limit; + // The first batch holds 101 documents unless asked for more; a page wants all of its rows. + if (options.batchSize) command.batchSize = options.batchSize; const result = await client.$runCommandRaw(command); return result?.cursor?.firstBatch || []; } diff --git a/packages/core/handlers/workers/integration-triggers.js b/packages/core/handlers/workers/integration-triggers.js new file mode 100644 index 000000000..d384605ae --- /dev/null +++ b/packages/core/handlers/workers/integration-triggers.js @@ -0,0 +1,25 @@ +/** + * The `{integration}Triggers` functions (ADR-055 §2-3): one per integration that declares triggers. Devtools gives + * each one schedule per distinct tick (`{ tick: '5m', time }`) and one per `at` trigger (`{ at, time }`); this only + * hands the event to the tick use case. + */ +const { createHandler } = require('@friggframework/core'); +const { loadAppDefinition } = require('../app-definition-loader'); +const { hasTriggers } = require('../../integrations/triggers'); +const { runTriggerTick } = require('../../integrations/use-cases/run-trigger-tick'); + +const handlers = {}; +const { integrations: integrationClasses } = loadAppDefinition(); + +for (const IntegrationClass of integrationClasses) { + if (!hasTriggers(IntegrationClass)) continue; + handlers[IntegrationClass.Definition.name] = { + tick: createHandler({ + eventName: `Trigger tick for ${IntegrationClass.Definition.name}`, + isUserFacingResponse: false, + method: async (event, context) => runTriggerTick(IntegrationClass, event || {}, context), + }), + }; +} + +module.exports = { handlers }; diff --git a/packages/core/handlers/workers/integration-triggers.test.js b/packages/core/handlers/workers/integration-triggers.test.js new file mode 100644 index 000000000..7389045b2 --- /dev/null +++ b/packages/core/handlers/workers/integration-triggers.test.js @@ -0,0 +1,28 @@ +/** + * The `{integration}Triggers` Lambda entry: one handler per integration that declares triggers, handing the + * schedule's event ({ tick | at, time }) to the tick use case. + */ +jest.mock('../../database/config', () => ({ DB_TYPE: 'mongodb', getDatabaseType: jest.fn(() => 'mongodb'), PRISMA_LOG_LEVEL: 'error,warn' })); + +class MockPoller { static Definition = { name: 'poller', triggers: { poll: { every: '15m', handler: 'poll' } } }; } +class MockDefined { static Definition = { name: 'defined', instanceTriggers: { tick: '5m' } }; } +class MockPlain { static Definition = { name: 'plain' }; } + +jest.mock('../app-definition-loader', () => ({ loadAppDefinition: () => ({ integrations: [MockPoller, MockDefined, MockPlain] }) })); +jest.mock('../../database/prisma', () => ({ connectPrisma: async () => {} })); +jest.mock('../../integrations/use-cases/run-trigger-tick', () => ({ runTriggerTick: jest.fn(async () => ({ sent: 3 })) })); + +const { handlers } = require('./integration-triggers'); +const { runTriggerTick } = require('../../integrations/use-cases/run-trigger-tick'); + +describe('integration-triggers handlers', () => { + it('exists only for integrations that declare triggers', () => { + expect(Object.keys(handlers).sort()).toEqual(['defined', 'poller']); + }); + + it('passes the schedule event and the Lambda context to the tick', async () => { + const context = { getRemainingTimeInMillis: () => 60000, callbackWaitsForEmptyEventLoop: true }; + await handlers.poller.tick({ tick: '15m', time: '2026-10-19T12:00:00Z' }, context); + expect(runTriggerTick).toHaveBeenCalledWith(MockPoller, { tick: '15m', time: '2026-10-19T12:00:00Z' }, expect.objectContaining({ getRemainingTimeInMillis: expect.any(Function) })); + }); +}); diff --git a/packages/core/infrastructure/scheduler/__tests__/recurrence.test.js b/packages/core/infrastructure/scheduler/__tests__/recurrence.test.js new file mode 100644 index 000000000..780025624 --- /dev/null +++ b/packages/core/infrastructure/scheduler/__tests__/recurrence.test.js @@ -0,0 +1,88 @@ +/** + * The host-neutral recurrence behind recurring triggers (ADR-055 + its ADR-063 amendment): what an interval or a + * five-field cron does, the floor an installation's choice is held to, and whether a cron fired between two ticks. + */ +const { + parseEvery, + minGapMinutes, + validateRecurrence, + recurrenceKey, + lastFiringIn, + wallClock, +} = require('../recurrence'); + +describe('recurrence (host-neutral)', () => { + it('reads every as minutes, hours or days and nothing else', () => { + expect([parseEvery('15m'), parseEvery('2h'), parseEvery('1d'), parseEvery(' 90 m ')]).toEqual([15, 120, 1440, 90]); + for (const bad of ['30s', '0m', '1w', '', null, 'm15', '1.5h']) expect(parseEvery(bad)).toBeNull(); + }); + + it('measures the shortest real gap of a cron, including the wrap into the next hour and day', () => { + expect(minGapMinutes({ cron: '*/15 * * * *' })).toBe(15); + expect(minGapMinutes({ cron: '*/7 * * * *' })).toBe(4); // :56 -> :00 + expect(minGapMinutes({ cron: '5,50 * * * *' })).toBe(15); // :50 -> :05 + expect(minGapMinutes({ cron: '0 */2 * * *' })).toBe(120); + expect(minGapMinutes({ cron: '0 9 * * 1-5' })).toBe(1440); + expect(minGapMinutes({ cron: '0 23,1 * * *' })).toBe(120); // 23:00 -> 01:00 next day + expect(minGapMinutes({ cron: '0 0 * * 0,7' })).toBe(10080); // 0 and 7 are both Sunday + expect(minGapMinutes({ every: '45m' })).toBe(45); + }); + + it('refuses what no host can run: bad fields, both day fields, a cron that never fires, an unknown time zone', () => { + expect(validateRecurrence({ cron: '0 9 * *' })).toEqual([expect.stringMatching(/five fields/)]); + expect(validateRecurrence({ cron: '60 * * * *' })).toEqual([expect.stringMatching(/minute.*out of range/)]); + expect(validateRecurrence({ cron: '0 9 1 * MON' })).toEqual(['Restrict day of month or day of week, not both']); + expect(validateRecurrence({ cron: '0 9 31 2 *' })).toEqual(['"0 9 31 2 *" never fires']); + expect(validateRecurrence({ cron: '0 9 * * *', timezone: 'Mars/Base' })).toEqual(['Unknown time zone "Mars/Base"']); + expect(validateRecurrence({ every: '1h', timezone: 'UTC' })).toEqual(['timezone applies to cron schedules only']); + expect(validateRecurrence({ every: '1h', cron: '0 * * * *' })).toEqual(['Give exactly one of every or cron']); + expect(validateRecurrence({ cron: '0 9 * * 1-5', timezone: 'America/New_York' })).toEqual([]); + }); + + it('holds a schedule to a floor by what it does, not how it is spelled', () => { + expect(validateRecurrence({ every: '10m' }, { min: '15m' })).toEqual(['Runs every 10 minutes at its most frequent; the minimum is 15m']); + expect(validateRecurrence({ cron: '0,5 * * * *' }, { min: '15m' })).toEqual(['Runs every 5 minutes at its most frequent; the minimum is 15m']); + expect(validateRecurrence({ cron: '*/15 * * * *' }, { min: '15m' })).toEqual([]); + expect(validateRecurrence({ every: '1h' }, { min: '1h' })).toEqual([]); + }); + + it('gives equal recurrences equal keys', () => { + expect(recurrenceKey({ every: '60m' })).toBe(recurrenceKey({ every: '1h' })); + expect(recurrenceKey({ cron: '0 9 * * *' })).toBe(recurrenceKey({ cron: '0 9 * * *', timezone: 'UTC' })); + expect(recurrenceKey({ cron: '0 9 * * *', timezone: 'Europe/London' })).not.toBe(recurrenceKey({ cron: '0 9 * * *' })); + }); +}); + +describe('lastFiringIn: did a cron fire since the previous tick', () => { + const at = (iso) => Date.parse(iso); + const iso = (ms) => (ms === null ? null : new Date(ms).toISOString()); + const NY = { cron: '0 8 * * 1-5', timezone: 'America/New_York' }; + + it('finds a firing in (from, to] in the cron\'s time zone, and only there', () => { + // Monday 2026-10-19 08:00 in New York is 12:00Z (EDT). + expect(iso(lastFiringIn(NY, at('2026-10-19T11:55:00Z'), at('2026-10-19T12:00:00Z')))).toBe('2026-10-19T12:00:00.000Z'); + expect(lastFiringIn(NY, at('2026-10-19T12:00:00Z'), at('2026-10-19T12:05:00Z'))).toBeNull(); // from is exclusive + expect(lastFiringIn(NY, at('2026-10-19T07:55:00Z'), at('2026-10-19T08:00:00Z'))).toBeNull(); // 08:00 UTC is not 08:00 NY + expect(lastFiringIn({ cron: '0 8 * * 1-5' }, at('2026-10-19T07:55:00Z'), at('2026-10-19T08:00:00Z'))).toBe(at('2026-10-19T08:00:00Z')); + expect(lastFiringIn(NY, at('2026-10-18T11:55:00Z'), at('2026-10-18T12:00:00Z'))).toBeNull(); // Sunday + }); + + it('returns the latest firing when several fall in one window, so a tick names one run', () => { + expect(iso(lastFiringIn({ cron: '*/2 * * * *' }, at('2026-10-19T12:00:00Z'), at('2026-10-19T12:05:30Z')))).toBe('2026-10-19T12:04:00.000Z'); + }); + + it('walks real instants across DST: a skipped wall-clock time does not fire, a repeated one fires twice', () => { + const twoThirty = { cron: '30 2 * * *', timezone: 'America/New_York' }; + // 2026-03-08: 02:00 EST jumps to 03:00 EDT, so 02:30 never happens. + expect(lastFiringIn(twoThirty, at('2026-03-08T06:00:00Z'), at('2026-03-08T08:00:00Z'))).toBeNull(); + // 2026-11-01: 01:30 happens at 05:30Z (EDT) and again at 06:30Z (EST). + const oneThirty = { cron: '30 1 * * *', timezone: 'America/New_York' }; + expect(iso(lastFiringIn(oneThirty, at('2026-11-01T05:00:00Z'), at('2026-11-01T06:00:00Z')))).toBe('2026-11-01T05:30:00.000Z'); + expect(iso(lastFiringIn(oneThirty, at('2026-11-01T06:00:00Z'), at('2026-11-01T07:00:00Z')))).toBe('2026-11-01T06:30:00.000Z'); + }); + + it('reads a wall clock in a zone into UTC fields, and an invalid cron as never firing', () => { + expect(wallClock(at('2026-10-19T12:34:56Z'), 'Asia/Kolkata').toISOString()).toBe('2026-10-19T18:04:00.000Z'); + expect(lastFiringIn({ cron: 'nope' }, 0, Date.now())).toBeNull(); + }); +}); diff --git a/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js b/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js deleted file mode 100644 index 6e669c11f..000000000 --- a/packages/core/infrastructure/scheduler/__tests__/recurring-schedules.test.js +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Recurring schedules: the host-neutral recurrence, its EventBridge translation, the adapters' upsert, and the - * reconcile command that keeps one installation's schedules in step with what it should have. - */ -const { - parseEvery, - minGapMinutes, - validateRecurrence, - recurrenceKey, -} = require('../recurrence'); -const { MockSchedulerAdapter } = require('../mock-scheduler-adapter'); -const { TICK, fillTick } = require('../scheduler-service-interface'); -const { createSchedulerCommands } = require('../../../application/commands/scheduler-commands'); - -jest.mock('@aws-sdk/client-scheduler', () => { - class Named extends Error { constructor(m) { super(m); this.name = this.constructor.name; } } - class ResourceNotFoundException extends Named {} - class ConflictException extends Named {} - const command = (type) => class { constructor(input) { this.type = type; this.input = input; } }; - return { - SchedulerClient: class { constructor() { this.sent = []; this.respond = async () => ({ ScheduleArn: 'arn:x' }); } async send(c) { this.sent.push(c); return this.respond(c); } }, - CreateScheduleCommand: command('create'), - UpdateScheduleCommand: command('update'), - DeleteScheduleCommand: command('delete'), - GetScheduleCommand: command('get'), - ResourceNotFoundException, - ConflictException, - }; -}); -const { ResourceNotFoundException, ConflictException } = require('@aws-sdk/client-scheduler'); -const { EventBridgeSchedulerAdapter, toEventBridgeExpression } = require('../eventbridge-scheduler-adapter'); - -const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/hubspot-queue'; -const QUEUE_ARN = 'arn:aws:sqs:us-east-1:123456789012:hubspot-queue'; - -describe('recurrence (host-neutral)', () => { - it('reads every as minutes, hours or days and nothing else', () => { - expect([parseEvery('15m'), parseEvery('2h'), parseEvery('1d'), parseEvery(' 90 m ')]).toEqual([15, 120, 1440, 90]); - for (const bad of ['30s', '0m', '1w', '', null, 'm15', '1.5h']) expect(parseEvery(bad)).toBeNull(); - }); - - it('measures the shortest real gap of a cron, including the wrap into the next hour and day', () => { - expect(minGapMinutes({ cron: '*/15 * * * *' })).toBe(15); - expect(minGapMinutes({ cron: '*/7 * * * *' })).toBe(4); // :56 -> :00 - expect(minGapMinutes({ cron: '5,50 * * * *' })).toBe(15); // :50 -> :05 - expect(minGapMinutes({ cron: '0 */2 * * *' })).toBe(120); - expect(minGapMinutes({ cron: '0 9 * * 1-5' })).toBe(1440); - expect(minGapMinutes({ cron: '0 23,1 * * *' })).toBe(120); // 23:00 -> 01:00 next day - expect(minGapMinutes({ cron: '0 0 * * 0,7' })).toBe(10080); // 0 and 7 are both Sunday - expect(minGapMinutes({ every: '45m' })).toBe(45); - }); - - it('refuses what no host can run: bad fields, both day fields, a cron that never fires, an unknown time zone', () => { - expect(validateRecurrence({ cron: '0 9 * *' })).toEqual([expect.stringMatching(/five fields/)]); - expect(validateRecurrence({ cron: '60 * * * *' })).toEqual([expect.stringMatching(/minute.*out of range/)]); - expect(validateRecurrence({ cron: '0 9 1 * MON' })).toEqual(['Restrict day of month or day of week, not both']); - expect(validateRecurrence({ cron: '0 9 31 2 *' })).toEqual(['"0 9 31 2 *" never fires']); - expect(validateRecurrence({ cron: '0 9 * * *', timezone: 'Mars/Base' })).toEqual(['Unknown time zone "Mars/Base"']); - expect(validateRecurrence({ every: '1h', timezone: 'UTC' })).toEqual(['timezone applies to cron schedules only']); - expect(validateRecurrence({ every: '1h', cron: '0 * * * *' })).toEqual(['Give exactly one of every or cron']); - expect(validateRecurrence({ cron: '0 9 * * 1-5', timezone: 'America/New_York' })).toEqual([]); - }); - - it('holds a schedule to a floor by what it does, not how it is spelled', () => { - expect(validateRecurrence({ every: '10m' }, { min: '15m' })).toEqual(['Runs every 10 minutes at its most frequent; the minimum is 15m']); - expect(validateRecurrence({ cron: '0,5 * * * *' }, { min: '15m' })).toEqual(['Runs every 5 minutes at its most frequent; the minimum is 15m']); - expect(validateRecurrence({ cron: '*/15 * * * *' }, { min: '15m' })).toEqual([]); - expect(validateRecurrence({ every: '1h' }, { min: '1h' })).toEqual([]); - }); - - it('gives equal recurrences equal keys, so a reconcile does not rewrite an unchanged schedule', () => { - expect(recurrenceKey({ every: '60m' })).toBe(recurrenceKey({ every: '1h' })); - expect(recurrenceKey({ cron: '0 9 * * *' })).toBe(recurrenceKey({ cron: '0 9 * * *', timezone: 'UTC' })); - expect(recurrenceKey({ cron: '0 9 * * *', timezone: 'Europe/London' })).not.toBe(recurrenceKey({ cron: '0 9 * * *' })); - }); - - it('fills tick placeholders deeply and leaves everything else alone', () => { - const out = fillTick({ a: TICK.id, b: [TICK.scheduledFor, 'x'], c: { d: TICK.id, e: 1 } }, { id: 'I', scheduledFor: 'T' }); - expect(out).toEqual({ a: 'I', b: ['T', 'x'], c: { d: 'I', e: 1 } }); - }); -}); - -describe('EventBridge translation', () => { - it('uses rate() in the largest whole unit and cron() with six fields, ? and weekday names', () => { - expect(toEventBridgeExpression({ every: '15m' })).toBe('rate(15 minutes)'); - expect(toEventBridgeExpression({ every: '60m' })).toBe('rate(1 hour)'); - expect(toEventBridgeExpression({ every: '2d' })).toBe('rate(2 days)'); - expect(toEventBridgeExpression({ every: '90m' })).toBe('rate(90 minutes)'); - expect(toEventBridgeExpression({ cron: '*/15 * * * *' })).toBe('cron(*/15 * * * ? *)'); - expect(toEventBridgeExpression({ cron: '0 0 1 * *' })).toBe('cron(0 0 1 * ? *)'); - // cron's 1-5 is Monday-Friday; EventBridge would read 1-5 as Sunday-Thursday, so names are written. - expect(toEventBridgeExpression({ cron: '0 9 * * 1-5' })).toBe('cron(0 9 ? * MON,TUE,WED,THU,FRI *)'); - expect(toEventBridgeExpression({ cron: '0 8 * JAN-MAR 0' })).toBe('cron(0 8 ? JAN-MAR SUN *)'); - }); - - it('throws on a recurrence it cannot express rather than creating a wrong schedule', () => { - expect(() => toEventBridgeExpression({ cron: '0 9 1 * MON' })).toThrow(/not both/); - }); -}); - -describe('EventBridgeSchedulerAdapter.upsertRecurring', () => { - let adapter; - beforeEach(() => { - process.env.SCHEDULER_ROLE_ARN = 'arn:aws:iam::123456789012:role/scheduler'; - adapter = new EventBridgeSchedulerAdapter({ region: 'us-east-1' }); - }); - - const recurring = { scheduleName: 'hubspot-int1-sync', recurrence: { cron: '0 9 * * 1-5', timezone: 'America/New_York' }, queueResourceId: QUEUE_ARN, payload: { event: 'SYNC', data: { tick: { id: TICK.id, scheduledFor: TICK.scheduledFor } } } }; - - it('updates an existing schedule: expression, zone, state, and per-firing context attributes in the input', async () => { - const out = await adapter.upsertRecurring({ ...recurring, enabled: false }); - expect(out).toEqual({ scheduledJobId: 'arn:x', created: false }); - expect(adapter.client.sent.map((c) => c.type)).toEqual(['update']); - const input = adapter.client.sent[0].input; - expect(input).toMatchObject({ Name: 'hubspot-int1-sync', ScheduleExpression: 'cron(0 9 ? * MON,TUE,WED,THU,FRI *)', ScheduleExpressionTimezone: 'America/New_York', State: 'DISABLED', FlexibleTimeWindow: { Mode: 'OFF' } }); - expect(input.ActionAfterCompletion).toBeUndefined(); // a recurring schedule must not delete itself - expect(JSON.parse(input.Target.Input).data.tick).toEqual({ id: '', scheduledFor: '' }); - }); - - it('creates it when it does not exist', async () => { - adapter.client.respond = async (c) => { if (c.type === 'update') throw new ResourceNotFoundException('nope'); return { ScheduleArn: 'arn:new' }; }; - expect(await adapter.upsertRecurring(recurring)).toEqual({ scheduledJobId: 'arn:new', created: true }); - expect(adapter.client.sent.map((c) => c.type)).toEqual(['update', 'create']); - expect(adapter.client.sent[1].input.State).toBe('ENABLED'); - }); - - it('updates after losing a create race to a concurrent writer', async () => { - let updates = 0; - adapter.client.respond = async (c) => { - if (c.type === 'update' && updates++ === 0) throw new ResourceNotFoundException('nope'); - if (c.type === 'create') throw new ConflictException('exists'); - return { ScheduleArn: 'arn:raced' }; - }; - expect(await adapter.upsertRecurring(recurring)).toEqual({ scheduledJobId: 'arn:raced', created: false }); - expect(adapter.client.sent.map((c) => c.type)).toEqual(['update', 'create', 'update']); - }); - - it('rate schedules always run in UTC; other errors propagate', async () => { - await adapter.upsertRecurring({ ...recurring, recurrence: { every: '1h' } }); - expect(adapter.client.sent[0].input).toMatchObject({ ScheduleExpression: 'rate(1 hour)', ScheduleExpressionTimezone: 'UTC' }); - adapter.client.respond = async () => { throw new Error('AccessDenied'); }; - await expect(adapter.upsertRecurring(recurring)).rejects.toThrow('AccessDenied'); - }); -}); - -describe('scheduler commands: recurring jobs', () => { - let service, commands; - const desired = (overrides = {}) => ({ key: 'SYNC', jobId: 'hubspot-int1-sync', recurrence: { every: '1h' }, event: 'SYNC', payload: { integrationId: 'int1' }, ...overrides }); - beforeEach(() => { - jest.spyOn(console, 'log').mockImplementation(() => {}); - jest.spyOn(console, 'error').mockImplementation(() => {}); - jest.spyOn(console, 'warn').mockImplementation(() => {}); - service = new MockSchedulerAdapter(); - commands = createSchedulerCommands({ integrationName: 'hubspot', schedulerService: service }); - }); - afterEach(() => jest.restoreAllMocks()); - - it('sends { event, data: { ...payload, tick } } to the queue ARN on every firing', async () => { - const res = await commands.upsertRecurringJob({ jobId: 'j1', recurrence: { every: '15m' }, event: 'POLL', payload: { integrationId: 'int1' }, queueUrl: QUEUE_URL }); - expect(res).toMatchObject({ jobId: 'j1', created: true }); - const stored = service._getSchedules().j1; - expect(stored.queueResourceId).toBe(QUEUE_ARN); - expect(service._simulateTrigger('j1', { id: 'exec-1', scheduledFor: '2026-10-18T09:00:00Z' })).toEqual({ - event: 'POLL', integrationName: 'hubspot', scheduleName: 'j1', - data: { integrationId: 'int1', tick: { id: 'exec-1', scheduledFor: '2026-10-18T09:00:00Z' } }, - }); - }); - - it('refuses an invalid recurrence as INVALID_JOB_DATA (400) without touching the host', async () => { - const res = await commands.upsertRecurringJob({ jobId: 'j1', recurrence: { every: '30s' }, event: 'POLL', queueUrl: QUEUE_URL }); - expect(res).toMatchObject({ error: 400, code: 'INVALID_JOB_DATA' }); - expect(service._getSchedules()).toEqual({}); - }); - - it('reconcile: creates, leaves an unchanged schedule alone, updates a changed one, deletes a dropped one', async () => { - const first = await commands.reconcileRecurringJobs({ desired: [desired(), desired({ key: 'DIGEST', jobId: 'hubspot-int1-digest', recurrence: { cron: '0 8 * * *' }, event: 'DIGEST' })], stored: {}, queueUrl: QUEUE_URL }); - expect(first).toMatchObject({ created: ['SYNC', 'DIGEST'], updated: [], deleted: [], failed: [] }); - expect(first.stored.SYNC).toMatchObject({ jobId: 'hubspot-int1-sync', recurrence: { every: '1h' }, event: 'SYNC', enabled: true }); - - const upsert = jest.spyOn(service, 'upsertRecurring'); - const second = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '60m' } })], stored: first.stored, queueUrl: QUEUE_URL }); - expect(second).toMatchObject({ unchanged: ['SYNC'], deleted: ['DIGEST'], created: [], updated: [] }); - expect(upsert).not.toHaveBeenCalled(); // 60m is 1h: nothing to write - expect(Object.keys(service._getSchedules())).toEqual(['hubspot-int1-sync']); - - const third = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '30m' }, enabled: false })], stored: second.stored, queueUrl: QUEUE_URL }); - expect(third.updated).toEqual(['SYNC']); - expect(service._getSchedules()['hubspot-int1-sync']).toMatchObject({ recurrence: { every: '30m' }, state: 'DISABLED' }); - expect(third.stored).toEqual({ SYNC: expect.objectContaining({ recurrence: { every: '30m' }, enabled: false }) }); - }); - - it('reconcile: a failed write leaves the stored entry as it was, so the next reconcile retries it', async () => { - const first = await commands.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); - jest.spyOn(service, 'upsertRecurring').mockRejectedValueOnce(new Error('throttled')); - const second = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '2h' } })], stored: first.stored, queueUrl: QUEUE_URL }); - expect(second.failed).toEqual([{ key: 'SYNC', reason: 'throttled' }]); - expect(second.stored).toEqual(first.stored); - const third = await commands.reconcileRecurringJobs({ desired: [desired({ recurrence: { every: '2h' } })], stored: second.stored, queueUrl: QUEUE_URL }); - expect(third.updated).toEqual(['SYNC']); - - jest.spyOn(service, 'deleteSchedule').mockRejectedValueOnce(new Error('denied')); - const fourth = await commands.reconcileRecurringJobs({ desired: [], stored: third.stored, queueUrl: QUEUE_URL }); - expect(fourth.failed).toEqual([{ key: 'SYNC', reason: 'denied' }]); - expect(fourth.stored).toEqual(third.stored); // still recorded: the host still holds it - }); - - it('reconcile: a renamed job is created under the new name and the old one deleted', async () => { - const first = await commands.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); - const second = await commands.reconcileRecurringJobs({ desired: [desired({ jobId: 'hubspot-int1-sync-v2' })], stored: first.stored, queueUrl: QUEUE_URL }); - expect(second.updated).toEqual(['SYNC']); - expect(Object.keys(service._getSchedules())).toEqual(['hubspot-int1-sync-v2']); - }); - - it('reconcile without a scheduler records nothing and says so', async () => { - const unconfigured = createSchedulerCommands({ integrationName: 'hubspot' }); - process.env.SCHEDULER_PROVIDER = 'nope'; - try { - const out = await unconfigured.reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); - expect(out).toMatchObject({ warning: 'Scheduler not configured', stored: {}, created: [] }); - } finally { delete process.env.SCHEDULER_PROVIDER; } - }); - - it('the methods work detached from the commands object', async () => { - const { reconcileRecurringJobs } = commands; - const out = await reconcileRecurringJobs({ desired: [desired()], stored: {}, queueUrl: QUEUE_URL }); - expect(out.created).toEqual(['SYNC']); - }); -}); diff --git a/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js b/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js index 63b988908..c06c46375 100644 --- a/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js +++ b/packages/core/infrastructure/scheduler/eventbridge-scheduler-adapter.js @@ -14,37 +14,12 @@ const { SchedulerClient, CreateScheduleCommand, - UpdateScheduleCommand, DeleteScheduleCommand, GetScheduleCommand, ResourceNotFoundException, - ConflictException, } = require('@aws-sdk/client-scheduler'); -const { SchedulerServiceInterface, TICK, fillTick } = require('./scheduler-service-interface'); -const { parseEvery, parseCron, validateRecurrence } = require('./recurrence'); - -const DAY_NAMES = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']; - -/** - * Host-neutral recurrence -> EventBridge Scheduler expression. - * { every: '15m' } -> rate(15 minutes) (whole hours and days use those units) - * { cron: 'm h dom mon dow' } -> cron(m h dom mon dow *), with `?` in whichever day field is unrestricted and the - * weekdays written as names, because EventBridge numbers them 1-7 from Sunday where cron numbers 0-6. - */ -function toEventBridgeExpression(recurrence) { - const problems = validateRecurrence(recurrence); - if (problems.length) throw new Error(`Invalid recurrence: ${problems.join('; ')}`); - if (recurrence.every !== undefined) { - const n = parseEvery(recurrence.every); - const [count, unit] = n % 1440 === 0 ? [n / 1440, 'day'] : n % 60 === 0 ? [n / 60, 'hour'] : [n, 'minute']; - return `rate(${count} ${unit}${count === 1 ? '' : 's'})`; - } - const parsed = parseCron(recurrence.cron); - const [minute, hour, dom, month] = parsed.parts; - const dow = parsed.restricted.dow ? [...parsed.fields[4]].map((d) => DAY_NAMES[d]).join(',') : '?'; - return `cron(${minute} ${hour} ${parsed.restricted.dow ? '?' : dom} ${month} ${dow} *)`; -} +const { SchedulerServiceInterface } = require('./scheduler-service-interface'); class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { constructor({ region } = {}) { @@ -121,45 +96,6 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { } } - /** - * Create or update a recurring schedule that sends `payload` to the queue on every firing. - * Update first (the common case once installed), create when it does not exist, and update again if a concurrent - * writer created it in between. - */ - async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled = true }) { - if (!scheduleName) throw new Error('scheduleName is required'); - if (!queueResourceId) throw new Error('queueResourceId is required'); - if (!this.roleArn) throw new Error('SCHEDULER_ROLE_ARN environment variable is not set'); - - const input = { - Name: scheduleName, - GroupName: this.scheduleGroupName, - ScheduleExpression: toEventBridgeExpression(recurrence), - ScheduleExpressionTimezone: recurrence.every !== undefined ? 'UTC' : recurrence.timezone || 'UTC', - FlexibleTimeWindow: { Mode: 'OFF' }, - State: enabled ? 'ENABLED' : 'DISABLED', - Target: { - Arn: queueResourceId, - RoleArn: this.roleArn, - // EventBridge substitutes these context attributes per invocation. - Input: JSON.stringify(fillTick(payload, { id: '', scheduledFor: '' })), - }, - }; - const update = async () => (await this.client.send(new UpdateScheduleCommand(input))).ScheduleArn; - try { - return { scheduledJobId: await update(), created: false }; - } catch (error) { - if (!(error instanceof ResourceNotFoundException)) throw error; - } - try { - const response = await this.client.send(new CreateScheduleCommand(input)); - return { scheduledJobId: response.ScheduleArn, created: true }; - } catch (error) { - if (error instanceof ConflictException) return { scheduledJobId: await update(), created: false }; - throw error; - } - } - /** * Delete a schedule * @@ -229,8 +165,6 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { exists: true, scheduledAt, state: response.State, - expression: response.ScheduleExpression, - timezone: response.ScheduleExpressionTimezone, }; } catch (error) { if (error instanceof ResourceNotFoundException) { @@ -247,4 +181,4 @@ class EventBridgeSchedulerAdapter extends SchedulerServiceInterface { } } -module.exports = { EventBridgeSchedulerAdapter, toEventBridgeExpression, TICK }; +module.exports = { EventBridgeSchedulerAdapter }; diff --git a/packages/core/infrastructure/scheduler/index.js b/packages/core/infrastructure/scheduler/index.js index 904573ee2..bbffae59b 100644 --- a/packages/core/infrastructure/scheduler/index.js +++ b/packages/core/infrastructure/scheduler/index.js @@ -1,8 +1,7 @@ /** * Scheduler Infrastructure * - * Provides scheduling capabilities for one-time and recurring jobs. Recurrence is host-neutral (recurrence.js); - * each adapter translates it. + * Provides scheduling capabilities for one-time jobs. * Follows hexagonal architecture with interface + adapters pattern. * * Providers: @@ -10,8 +9,7 @@ * - mock: In-memory mock scheduler (local development) */ -const { SchedulerServiceInterface, TICK, fillTick } = require('./scheduler-service-interface'); -const recurrence = require('./recurrence'); +const { SchedulerServiceInterface } = require('./scheduler-service-interface'); const { EventBridgeSchedulerAdapter } = require('./eventbridge-scheduler-adapter'); const { MockSchedulerAdapter } = require('./mock-scheduler-adapter'); const { @@ -23,9 +21,6 @@ const { module.exports = { // Interface (Port) SchedulerServiceInterface, - TICK, - fillTick, - ...recurrence, // Adapters EventBridgeSchedulerAdapter, diff --git a/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js b/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js index 51970be77..9826e4934 100644 --- a/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js +++ b/packages/core/infrastructure/scheduler/mock-scheduler-adapter.js @@ -7,8 +7,7 @@ * This adapter implements SchedulerServiceInterface for local development and testing. */ -const { SchedulerServiceInterface, fillTick } = require('./scheduler-service-interface'); -const { validateRecurrence } = require('./recurrence'); +const { SchedulerServiceInterface } = require('./scheduler-service-interface'); class MockSchedulerAdapter extends SchedulerServiceInterface { constructor(options = {}) { @@ -62,28 +61,6 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { }; } - /** - * Create or update a recurring schedule (kept in memory; `_simulateTrigger` fires it). - */ - async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled = true }) { - if (!scheduleName) throw new Error('scheduleName is required'); - if (!queueResourceId) throw new Error('queueResourceId is required'); - const problems = validateRecurrence(recurrence); - if (problems.length) throw new Error(`Invalid recurrence: ${problems.join('; ')}`); - const created = !this.schedules.has(scheduleName); - this.schedules.set(scheduleName, { - scheduleName, - recurrence, - queueResourceId, - payload, - recurring: true, - createdAt: this.schedules.get(scheduleName)?.createdAt || new Date().toISOString(), - state: enabled ? 'ENABLED' : 'DISABLED', - }); - if (this.verbose) console.log(`[MockScheduler] ${created ? 'Created' : 'Updated'} recurring schedule: ${scheduleName}`, recurrence); - return { scheduledJobId: `mock-job-${scheduleName}`, created }; - } - /** * Delete a scheduled job * @@ -122,7 +99,6 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { exists: true, scheduledAt: schedule.scheduledAt, state: schedule.state, - ...(schedule.recurrence ? { recurrence: schedule.recurrence } : {}), }; } @@ -150,7 +126,7 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { * @param {string} scheduleName - Name of the schedule to trigger * @returns {Object|null} The payload that would be sent, or null if not found */ - _simulateTrigger(scheduleName, tick = { id: `mock-${Date.now()}`, scheduledFor: new Date().toISOString() }) { + _simulateTrigger(scheduleName) { const schedule = this.schedules.get(scheduleName); if (!schedule) { console.log(`[MockScheduler] Cannot trigger - schedule not found: ${scheduleName}`); @@ -158,10 +134,9 @@ class MockSchedulerAdapter extends SchedulerServiceInterface { } console.log(`[MockScheduler] Simulating trigger for: ${scheduleName}`); - const payload = schedule.recurring ? fillTick(schedule.payload, tick) : schedule.payload; - console.log(`[MockScheduler] Payload:`, JSON.stringify(payload, null, 2)); + console.log(`[MockScheduler] Payload:`, JSON.stringify(schedule.payload, null, 2)); - return payload; + return schedule.payload; } } diff --git a/packages/core/infrastructure/scheduler/recurrence.js b/packages/core/infrastructure/scheduler/recurrence.js index 4cccf95f4..1e15ba1f4 100644 --- a/packages/core/infrastructure/scheduler/recurrence.js +++ b/packages/core/infrastructure/scheduler/recurrence.js @@ -1,11 +1,12 @@ /** - * Recurrence: how often a recurring job fires, in a form no host owns. + * Recurrence: how often something repeats, in a form no host owns. * - * { every: '15m' } every 15 minutes ('30s' is refused: hosts tick in minutes) + * { every: '15m' } every 15 minutes ('30s' is refused: ticks are whole minutes) * { cron: '0 9 * * 1-5', timezone: 'America/New_York' } standard five-field cron (minute hour dom month dow) * - * Each scheduler adapter translates this into its host's syntax (EventBridge: rate(...) / cron(...) with six fields - * and `?`). Keeping the neutral form here is what lets one definition, or one integration, run on any host. + * Recurring triggers (ADR-055, integrations/triggers.js) use it for the timing an installation may choose and for + * the floor a developer sets on that choice. Nothing here creates a host schedule: the tick asks `lastFiringIn` + * whether a cron fired since the previous tick, so a per-installation calendar needs no schedule of its own. * * Pure; no I/O. */ @@ -166,4 +167,38 @@ function recurrenceKey(r) { return r.every !== undefined ? `every:${parseEvery(r.every)}` : `cron:${String(r.cron).trim().split(/\s+/).join(' ')}@${r.timezone || 'UTC'}`; } -module.exports = { parseEvery, parseCron, minGapMinutes, validateRecurrence, recurrenceKey, isTimeZone, matches }; +const wallClockFormats = new Map(); +/** + * The wall-clock time of an instant in a time zone, as a Date whose UTC fields hold that wall clock (what `matches` + * reads). Seconds are dropped. + */ +function wallClock(ms, timezone = 'UTC') { + if (timezone === 'UTC') return new Date(Math.floor(ms / 60000) * 60000); + let format = wallClockFormats.get(timezone); + if (!format) { + format = new Intl.DateTimeFormat('en-US', { timeZone: timezone, hourCycle: 'h23', year: 'numeric', month: 'numeric', day: 'numeric', hour: 'numeric', minute: 'numeric' }); + wallClockFormats.set(timezone, format); + } + const p = Object.fromEntries(format.formatToParts(new Date(ms)).map((x) => [x.type, x.value])); + return new Date(Date.UTC(Number(p.year), Number(p.month) - 1, Number(p.day), Number(p.hour) % 24, Number(p.minute))); +} + +/** + * The latest whole minute in (fromMs, toMs] at which a cron recurrence fires in its time zone, as epoch ms, or null. + * + * This is how one shared tick serves a calendar each installation chose: the tick asks whether the cron fired since + * the previous tick. It is stateless, so a tick that runs twice for the same slot gives the same answer (and the same + * `scheduledFor`, which is what a run dedupes on). Real instants are walked, so a wall-clock time a DST change skips + * does not fire that day, and one it repeats fires twice. + */ +function lastFiringIn(recurrence, fromMs, toMs) { + const parsed = parseCron(recurrence?.cron); + if (parsed.error) return null; + const tz = recurrence.timezone || 'UTC'; + for (let t = Math.floor(toMs / 60000) * 60000; t > fromMs; t -= 60000) { + if (matches(parsed, wallClock(t, tz))) return t; + } + return null; +} + +module.exports = { parseEvery, parseCron, minGapMinutes, validateRecurrence, recurrenceKey, isTimeZone, matches, wallClock, lastFiringIn }; diff --git a/packages/core/infrastructure/scheduler/scheduler-service-interface.js b/packages/core/infrastructure/scheduler/scheduler-service-interface.js index 5220f2246..5972593b8 100644 --- a/packages/core/infrastructure/scheduler/scheduler-service-interface.js +++ b/packages/core/infrastructure/scheduler/scheduler-service-interface.js @@ -23,25 +23,6 @@ class SchedulerServiceInterface { throw new Error('Method scheduleOneTime must be implemented by subclass'); } - /** - * Create or update a recurring schedule (idempotent by name). - * - * The recurrence is host-neutral ({ every } or { cron, timezone }, see recurrence.js); each adapter translates it. - * String values in `payload` equal to TICK.id or TICK.scheduledFor are replaced, per firing, with that firing's - * id and scheduled time when the host can supply them, so a consumer can dedupe redeliveries of one tick. - * - * @param {Object} params - * @param {string} params.scheduleName - Unique name for the schedule - * @param {{every?: string, cron?: string, timezone?: string}} params.recurrence - * @param {string} params.queueResourceId - Queue resource identifier to send message to - * @param {Object} params.payload - JSON payload to send on every firing - * @param {boolean} [params.enabled=true] - false keeps the schedule but stops it firing - * @returns {Promise<{scheduledJobId: string, created: boolean}>} - */ - async upsertRecurring({ scheduleName, recurrence, queueResourceId, payload, enabled }) { - throw new Error('Method upsertRecurring must be implemented by subclass'); - } - /** * Delete a scheduled job * @@ -63,19 +44,4 @@ class SchedulerServiceInterface { } } -/** Placeholders an adapter fills per firing (see upsertRecurring). */ -const TICK = Object.freeze({ id: '{{frigg.tick.id}}', scheduledFor: '{{frigg.tick.scheduledFor}}' }); - -/** Replace every string equal to a TICK placeholder, deeply. */ -function fillTick(payload, values) { - const swap = (v) => { - if (v === TICK.id) return values.id; - if (v === TICK.scheduledFor) return values.scheduledFor; - if (Array.isArray(v)) return v.map(swap); - if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, swap(x)])); - return v; - }; - return swap(payload); -} - -module.exports = { SchedulerServiceInterface, TICK, fillTick }; +module.exports = { SchedulerServiceInterface }; diff --git a/packages/core/integrations/data-defined/README.md b/packages/core/integrations/data-defined/README.md index 5d89ae4c0..4a9b6ca51 100644 --- a/packages/core/integrations/data-defined/README.md +++ b/packages/core/integrations/data-defined/README.md @@ -25,7 +25,7 @@ installation (Integration.config) ─┘ │ | `steps.js`, `interpreter.js`, `transforms.js` | The step catalogue and the interpreter | | `validate.js` | Structure + cross-references against the manifests (steps, templates, views, settings, `$.install`) | | `manifests.js` | `validateManifest`: the manifests themselves (kinds, option lists, account fields) | -| `compile.js` | Definition + installation → events, webhook routes, schedules, user actions | +| `compile.js` | Definition + installation → events, webhook routes, recurring triggers, user actions | | `runtime.js` | Ports (Api calls, link store), `runWorkflow`, `runQuery`, `renderSurface`, `runLifecycle`, `resolveOptions`, `installContext` | | `versioning.js` | `diff`, `checkBump`, `planUpgrade` | | `data-defined-integration.js` | `defineIntegrationClass`: the IntegrationBase subclass that runs it all | @@ -98,17 +98,21 @@ version cannot both win. With `memoryDefinitions()` (definitions bundled in the ## Schedules -A schedule-triggered workflow or a polling sync gets one recurring host schedule per installation, through the same -core mechanism hand-written integrations use (`Definition.schedules`; ADR-063 §15a). Mark one `configurable` to let -installations pick their own timing, optionally with a floor: +A schedule-triggered workflow or a polling sync is a recurring trigger (ADR-055), the same mechanism hand-written +integrations declare as `Definition.triggers` (ADR-063 §15a). Nothing is scheduled per installation: the class +declares `instanceTriggers: { tick: '5m' }`, so devtools provisions one tick function for it, and each tick compiles +every ENABLED installation's pinned definition with its config and queues the runs that are due. Mark one +`configurable` to let installations pick their own timing, optionally with a floor: ```json "trigger": { "type": "schedule", "cron": "0 16 * * 1-5", "timezone": "America/New_York", "configurable": { "min": "1h" } } "ongoing": { "mode": "poll", "every": "15m", "configurable": true } ``` -The choice goes in `workflows..schedule` / `syncs..schedule` (the configure form offers the field). Create, -update, upgrade and delete keep the host in step. What it holds is recorded at `config.runtime.schedules`. +The choice goes in `workflows..schedule` / `syncs..schedule` (the configure form offers the field). An +interval runs on whole multiples of the 5-minute tick; a cron runs on the first tick after it fires. Create, update, +upgrade and delete need no scheduler call: the next tick reads the new state. A run is deduped on +`tick::`, so a redelivered run happens once. ## Steps (v0.1) diff --git a/packages/core/integrations/data-defined/compile.js b/packages/core/integrations/data-defined/compile.js index eefbcecff..b90e9a76a 100644 --- a/packages/core/integrations/data-defined/compile.js +++ b/packages/core/integrations/data-defined/compile.js @@ -2,7 +2,7 @@ * Compile a definition + one installation into what Frigg runs: an events map shaped exactly like a hand-written * integration's `this.events`, plus the routing tables the runtime needs. Pure; no I/O. * - * Only what the installation has enabled is compiled, so a disabled workflow has no event, no route and no schedule. + * Only what the installation has enabled is compiled, so a disabled workflow has no event, no route and no trigger. * * Event names: SYNC__BACKFILL (USER_ACTION) · SYNC__PAGE (QUEUE) · SYNC__ONGOING (WEBHOOK|CRON) * WF_ (WEBHOOK|CRON|USER_ACTION|QUEUE) · Q_ (ROUTE) · UI_ (ROUTE) @@ -17,8 +17,7 @@ function isOn(item, chosen) { function compile(def, installation = {}) { const events = {}; const webhooks = {}; // moduleKey -> topic -> [eventName] - const schedules = []; // { key, event, default: { every } | { cron, timezone }, configurable, choice } (integrations/schedules.js) - const configurableOf = (c) => (c ? (c === true ? {} : c) : false); + const triggers = []; // recurring triggers (integrations/triggers.js): { name, every | cron, timezone?, configurable, choice } const userActions = []; // { event, label, from } const routes = []; // { method, path, event } const add = (name, e) => { events[name] = e; return name; }; @@ -40,7 +39,7 @@ function compile(def, installation = {}) { } } else if (mode === 'poll') { const n = add(eventName('SYNC', s.id, 'ONGOING'), { type: 'CRON', ref: { kind: 'sync', id: s.id, phase: 'ongoing' } }); - schedules.push({ key: n, event: n, default: { every: s.ongoing.every || '15m' }, configurable: configurableOf(s.ongoing.configurable), choice: installation.syncs?.[s.id]?.schedule }); + triggers.push({ name: n, every: s.ongoing.every || '15m', configurable: s.ongoing.configurable || false, choice: installation.syncs?.[s.id]?.schedule }); } } @@ -51,8 +50,8 @@ function compile(def, installation = {}) { const n = add(eventName('WF', w.id), { type, ...(type === 'USER_ACTION' ? { userActionType: 'DATA_ACTION', title: t.label } : {}), ref: { kind: 'workflow', id: w.id } }); if (t.type === 'webhook') route(t.module, t.topic, n); if (t.type === 'schedule') { - const recurrence = t.every !== undefined ? { every: t.every } : { cron: t.cron, ...(t.timezone ? { timezone: t.timezone } : {}) }; - schedules.push({ key: n, event: n, default: recurrence, configurable: configurableOf(t.configurable), choice: installation.workflows?.[w.id]?.schedule }); + const timing = t.every !== undefined ? { every: t.every } : { cron: t.cron, ...(t.timezone ? { timezone: t.timezone } : {}) }; + triggers.push({ name: n, ...timing, configurable: t.configurable || false, choice: installation.workflows?.[w.id]?.schedule }); } if (t.type === 'userAction') userActions.push({ event: n, label: t.label, from: t.from || null }); if (t.type === 'syncRecord') route('$sync', `${t.sync}:${(t.on || []).join('|')}`, n); @@ -65,7 +64,7 @@ function compile(def, installation = {}) { routes.push({ method: 'GET', path: `/ui/${u.id}`, event: n }); } - return { events, webhooks, schedules, userActions, routes }; + return { events, webhooks, triggers, userActions, routes }; } /** Which compiled events a delivered webhook goes to. Topic patterns may end in ".*". */ diff --git a/packages/core/integrations/data-defined/data-defined-integration.js b/packages/core/integrations/data-defined/data-defined-integration.js index 1d7e10dbe..ee44340ae 100644 --- a/packages/core/integrations/data-defined/data-defined-integration.js +++ b/packages/core/integrations/data-defined/data-defined-integration.js @@ -21,6 +21,22 @@ const { makePorts, runWorkflow, runQuery, renderSurface, runLifecycle, resolveOp const { eventsIn, handshakeFor, modulesFor } = require('./webhooks'); const { buildConfigForm, missingRequired } = require('./config-form'); +/** The tick every data-defined trigger runs on: a chosen interval is rounded up to whole multiples of it. */ +const INSTANCE_TICK = '5m'; + +/** A compiled definition's triggers as IntegrationBase trigger declarations: each runs its compiled event. */ +function triggersOf(def, config) { + return compile(def, config).triggers.map((t) => ({ + ...t, + tick: INSTANCE_TICK, + handler(payload) { + const event = this.events?.[t.name]; + if (!event) throw new Error(`Trigger "${t.name}" has no compiled event in this installation`); + return event.handler({ data: { integrationId: this.id, ...payload } }); + }, + })); +} + /** * Shape A — one class per definition id (defineIntegrationClass): the class name and modules come from the * definition. Publishing a new VERSION deploys nothing; a new definition ID is a new class (a deploy, and its own @@ -80,9 +96,35 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati let runStore = runs; const runRepository = () => (runStore === undefined ? (runStore = require('./repositories/workflow-run-repository').createWorkflowRunRepository()) : runStore); const publicBase = () => `${process.env.FRIGG_PUBLIC_URL || ''}/api/${id}-integration/webhooks`; + const pinnedDefinitions = new Map(); // `${id}@${version}` -> definition, for the trigger tick class DataDefinedIntegration extends Base { - static Definition = { name, version, modules, webhooks: { enabled: true }, display }; // version is display only; installations pin their own + // version is display only; installations pin their own. instanceTriggers: schedule-triggered workflows and + // polling syncs are this class's recurring triggers, resolved per installation (triggersFor) on one 5m tick. + static Definition = { name, version, modules, webhooks: { enabled: true }, display, instanceTriggers: { tick: INSTANCE_TICK } }; + + /** + * The recurring triggers of one installation (ADR-055): its pinned definition compiled with its config, so a + * switched-off workflow has none and a configurable one carries the installation's choice. Each runs its + * compiled event. Read by the shared tick from the record alone (no modules, no credentials). + */ + static async triggersFor({ config = {} } = {}) { + const pin = config.definition || (defaultId ? { id: defaultId } : null); + if (!pin?.id) return []; + let def; + if (pin.version) { + // Published versions are immutable, so a tick over many installations reads each one once. + const key = `${pin.id}@${pin.version}`; + if (!pinnedDefinitions.has(key)) { + if (pinnedDefinitions.size >= 100) pinnedDefinitions.clear(); + pinnedDefinitions.set(key, await definitions.get(pin.id, pin.version, { checksum: pin.checksum })); + } + def = pinnedDefinitions.get(key); + } else { + def = await definitions.latest(pin.id); + } + return def ? triggersOf(def, config) : []; + } /** How the routers find data-defined classes and the store and manifests they run on. */ static DataDefined = { definitions, manifests, shared: !defaultId, runs: () => runRepository() }; @@ -150,10 +192,9 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati ? await definitions.get(this.def.id, msg.definitionVersion) || this.def : this.def; const installation = this.installation(); - // A scheduled tick may be delivered twice; its scheduled time makes the dedupe id (an unfilled - // placeholder, from a host that cannot supply it, is not an id). - const scheduledFor = msg.tick?.scheduledFor && !String(msg.tick.scheduledFor).startsWith('{{') ? msg.tick.scheduledFor : null; - const eventId = msg.eventId || (scheduledFor ? `tick:${msg.scheduleKey}:${scheduledFor}` : undefined); + // A trigger run may be delivered twice (a redelivery, two ticks in one slot); the trigger and the time it + // was scheduled for make its dedupe id. + const eventId = msg.eventId || (msg.trigger && msg.scheduledFor ? `tick:${msg.trigger}:${msg.scheduledFor}` : undefined); if (ref.kind === 'workflow') return this.recordRun(await runWorkflow({ def, installation, workflowId: ref.id, event: msg.body ?? msg, eventId, record: msg.record, ports: this.ports, manifests })); if (ref.kind === 'query') return runQuery({ def, installation, queryId: ref.id, record: msg.record, ports: this.ports }); if (ref.kind === 'ui') return renderSurface({ def, installation, surfaceId: ref.id, record: payload.req?.body?.record || msg.record, ports: this.ports }); @@ -256,15 +297,9 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati return super.onDelete(params); } - /** - * The recurring schedules this installation should have: the definition's schedule-triggered workflows and - * polling syncs that are switched on, with the installation's timing where the definition allows it. - * Compiled from the current config every time, so the base class's onUpdate (which patches config, then - * reconciles) schedules what was just chosen. The base class's onCreate, onUpdate and onDelete keep the host - * in step (IntegrationBase.syncSchedules). - */ - scheduleEntries() { - return this.def ? compile(this.def, this.config || {}).schedules : []; + /** This installation's triggers, from the definition it already loaded and its current config. */ + async triggerDeclarations() { + return this.def ? triggersOf(this.def, this.config || {}) : this.constructor.triggersFor({ config: this.config || {} }); } /** The configure form (GET /api/integrations/:id/config/options), generated from the definition. */ @@ -336,9 +371,8 @@ function makeClass({ name, defaultId, definitions, manifests, runCode, Integrati await this.patchConfig(config); this.config = { ...this.config, ...config }; await this.loadDefinition(); - // The new version may add, drop or retime schedules. - const schedules = typeof this.syncSchedules === 'function' ? await this.syncSchedules() : null; - return { ...plan, config, applied: true, ...(schedules ? { schedules } : {}) }; + // Triggers need nothing here: the next tick reads the new pin and config. + return { ...plan, config, applied: true }; } async recordRun(run) { diff --git a/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json b/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json index cf93da323..29469f884 100644 --- a/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json +++ b/packages/core/integrations/data-defined/schema/installation.v0.1.schema.json @@ -77,55 +77,18 @@ }, "runtime": { "type": "object", - "description": "Owned by the runtime, not the customer. installed: each onInstall step's result by step id (webhook subscriptions), read by onUninstall as $.install.runtime.installed.. schedules: the recurring schedules the host holds for this installation, by event (written by IntegrationBase.syncSchedules). Sync cursors join it with the sync engine.", + "description": "Owned by the runtime, not the customer. installed: each onInstall step's result by step id (webhook subscriptions), read by onUninstall as $.install.runtime.installed.. Sync cursors join it with the sync engine.", "additionalProperties": false, "properties": { "installed": { "type": "object" - }, - "schedules": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "jobId", - "recurrence", - "event" - ], - "properties": { - "jobId": { - "type": "string" - }, - "recurrence": { - "type": "object" - }, - "event": { - "type": "string" - }, - "enabled": { - "type": "boolean" - }, - "jobArn": { - "type": [ - "string", - "null" - ] - }, - "fingerprint": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } - } } } } }, "$defs": { "scheduleChoice": { - "description": "The installation's timing for a schedule the definition marks configurable: every, or cron with an optional time zone. Honoured only within the definition's minimum; otherwise the definition's schedule runs.", + "description": "The installation's timing for a schedule trigger or polling sync the definition marks configurable: every, or cron with an optional time zone. Honoured only within the definition's minimum; otherwise the definition's timing runs. The integration's shared trigger tick reads it (ADR-055); nothing is scheduled per installation.", "type": "object", "additionalProperties": false, "properties": { diff --git a/packages/core/integrations/data-defined/tests/data-defined.test.js b/packages/core/integrations/data-defined/tests/data-defined.test.js index a7ec6229f..d783484a3 100644 --- a/packages/core/integrations/data-defined/tests/data-defined.test.js +++ b/packages/core/integrations/data-defined/tests/data-defined.test.js @@ -73,7 +73,7 @@ describe('compile', () => { expect(c.events.WF_WON_THANK_YOU).toBeUndefined(); // default off, left off expect(c.events.SYNC_CONTACTS_BACKFILL).toMatchObject({ type: 'USER_ACTION', userActionType: 'SYNC_ACTION' }); expect(c.events.WF_TEXT_FROM_CARD).toMatchObject({ type: 'USER_ACTION' }); - expect(c.schedules).toEqual([]); // stale-contacts is off + expect(c.triggers).toEqual([]); // stale-contacts is off expect(d.routeWebhook(c, 'quo', 'call.completed')).toEqual(['WF_LOG_CALL']); expect(d.routeWebhook(c, 'hubspot', 'contact.propertyChange')).toEqual(['SYNC_CONTACTS_ONGOING']); expect(c.routes.map((r) => r.path)).toEqual(['/ui/quo-card', '/ui/deals-panel']); diff --git a/packages/core/integrations/data-defined/tests/schedules.test.js b/packages/core/integrations/data-defined/tests/schedules.test.js index 5ef0ed89d..c3fc76913 100644 --- a/packages/core/integrations/data-defined/tests/schedules.test.js +++ b/packages/core/integrations/data-defined/tests/schedules.test.js @@ -1,7 +1,7 @@ /** - * Recurring schedules for data-defined installations: what a definition may declare, what an installation may - * change, and that the class keeps one host schedule per installation in step through create, update, upgrade and - * delete (IntegrationBase.syncSchedules over the real scheduler commands and the mock adapter). + * Schedules for data-defined installations, as ADR-055 recurring triggers: what a definition may declare, what an + * installation may change, and that one shared tick (RunTriggerTick over the class's triggersFor) finds each + * installation's runs due from its pinned definition and config, with no scheduler call anywhere in the lifecycle. */ jest.mock('../../../database/config', () => ({ DB_TYPE: 'mongodb', getDatabaseType: () => 'mongodb', PRISMA_LOG_LEVEL: 'error,warn' })); @@ -11,8 +11,9 @@ const { validate } = require('../validate'); const { diff, planUpgrade } = require('../versioning'); const { buildConfigForm } = require('../config-form'); const { IntegrationBase } = require('../../integration-base'); -const { MockSchedulerAdapter } = require('../../../infrastructure/scheduler/mock-scheduler-adapter'); -const { createSchedulerCommands } = require('../../../application/commands/scheduler-commands'); +const { RunTriggerTick } = require('../../use-cases/run-trigger-tick'); +const { IntegrationEventDispatcher } = require('../../../handlers/integration-event-dispatcher'); +const { provisioningOf } = require('../../triggers'); const manifests = d.loadManifests(); const example = (id) => JSON.parse(JSON.stringify(d.loadExamples().find((x) => x.id === id))); @@ -28,18 +29,18 @@ function digestDef({ configurable = { min: '1h' }, version } = {}) { } describe('definition and installation', () => { - it('compiles each switched-on schedule into an entry with its default, its allowance and the installation\'s choice', () => { + it('compiles each switched-on schedule into a trigger with its timing, its allowance and the installation\'s choice', () => { const def = digestDef(); const out = compile(def, { workflows: { 'due-tomorrow-digest': { schedule: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } } } }); - expect(out.schedules).toEqual([{ key: DIGEST, event: DIGEST, default: { cron: '0 16 * * 1-5', timezone: 'America/New_York' }, configurable: { min: '1h' }, choice: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } }]); - expect(compile(def, { workflows: { 'due-tomorrow-digest': { enabled: false } } }).schedules).toEqual([]); + expect(out.triggers).toEqual([{ name: DIGEST, cron: '0 16 * * 1-5', timezone: 'America/New_York', configurable: { min: '1h' }, choice: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } }]); + expect(compile(def, { workflows: { 'due-tomorrow-digest': { enabled: false } } }).triggers).toEqual([]); }); it('a polling sync compiles to an interval entry, configurable when the definition says so', () => { const def = example('pipedrive-hubspot'); def.syncs[0].ongoing = { mode: 'poll', every: '30m', configurable: true }; const out = compile(def, { syncs: { [def.syncs[0].id]: { schedule: { every: '2h' } } } }); - expect(out.schedules).toEqual([expect.objectContaining({ default: { every: '30m' }, configurable: {}, choice: { every: '2h' } })]); + expect(out.triggers).toEqual([expect.objectContaining({ every: '30m', configurable: true, choice: { every: '2h' } })]); }); it('validates a schedule as a host would run it, and a default that breaks its own floor', () => { @@ -81,18 +82,16 @@ describe('definition and installation', () => { }); }); -describe('installation lifecycle', () => { - let service; - - /** IntegrationBase's real lifecycle and syncSchedules, with persistence and module wiring stubbed. */ +describe('one shared tick for every installation', () => { + /** IntegrationBase's real lifecycle and trigger run, with persistence and module wiring stubbed. */ class Base extends IntegrationBase { constructor(p = {}) { super(p); - this.id = p.id; this.config = p.config; this.status = 'IN_CREATION'; + this.id = p.id; this.config = p.config; this.status = p.status || 'IN_CREATION'; this.updateIntegrationStatus = { execute: async () => true }; this.updateIntegrationMessages = { execute: jest.fn(async () => true) }; this.patchIntegrationConfig = { execute: async (id, patch) => ({ config: (this.config = { ...this.config, ...patch }) }) }; - this.schedulerCommands = createSchedulerCommands({ integrationName: 'data-defined', schedulerService: service }); + this.triggerProcessRepository = { findByIntegrationAndType: async () => [], create: async (r) => ({ id: 'p1', ...r }), update: async () => ({}) }; this.ran = []; } async initialize() { this.registerEventHandlers(); } @@ -103,76 +102,108 @@ describe('installation lifecycle', () => { async deleteMapping() {} } + const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/data-defined-queue'; const setup = (defs) => { const definitions = d.memoryDefinitions(defs); const Cls = d.defineSharedIntegrationClass({ definitions, manifests, modules: { asana: {}, slack: {} }, IntegrationBase: Base, runs: false }); - return (config) => { const inst = new Cls({ id: 'int9', config: { type: 'data-defined', ...config } }); const api = new Proxy({}, { get: () => async () => ({ id: 'X1' }) }); // install steps succeed - inst.asana = { api }; inst.slack = { api }; return inst; }; + const install = (id, config, status) => { + const inst = new Cls({ id, status, config: { type: 'data-defined', ...config } }); + const api = new Proxy({}, { get: () => async () => ({ id: 'X1' }) }); // install steps succeed + inst.asana = { api }; inst.slack = { api }; + return inst; + }; + /** The tick over the installations' records, as the Lambda runs it (fake repository and queue). */ + const tickAt = async (installations, time) => { + const sent = []; + const integrationRepository = { findIntegrations: async ({ status }) => installations.filter((i) => i.status === status).map((i) => ({ id: i.id, status: i.status, config: i.config })) }; + const queue = { batchSend: async (entries) => { sent.push(...entries); }, send: async () => {} }; + await new RunTriggerTick({ integrationClass: Cls, integrationRepository, queue, queueUrl: QUEUE_URL }).execute({ tick: '5m', time }); + return sent; + }; + return { Cls, install, tickAt }; }; + // The digest runs at 16:00 New York on weekdays: 20:00Z on Monday 2026-10-19 (EDT). + const DUE = '2026-10-19T20:00:00Z'; + + beforeEach(() => { jest.spyOn(console, 'log').mockImplementation(() => {}); }); + afterEach(() => { jest.restoreAllMocks(); }); - beforeEach(() => { - jest.spyOn(console, 'log').mockImplementation(() => {}); - process.env.DATA_DEFINED_QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/data-defined-queue'; - service = new MockSchedulerAdapter(); + it('the class declares one 5m tick for all its installations, and nothing per installation', () => { + const { Cls } = setup([digestDef()]); + expect(provisioningOf(Cls)).toEqual({ ticks: [5], at: [] }); }); - afterEach(() => { jest.restoreAllMocks(); delete process.env.DATA_DEFINED_QUEUE_URL; }); - it('create schedules the installation\'s workflows; an update to its own timing retimes it; delete removes it', async () => { - const newInstall = setup([digestDef()]); - const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version } }); + it('create, update and delete schedule nothing; the tick reads the current config each time', async () => { + const scheduler = require('../../../application/commands/scheduler-commands'); + const spy = jest.spyOn(scheduler, 'createSchedulerCommands'); + const { install, tickAt } = setup([digestDef()]); + const inst = install('int9', { definition: { id: 'asana-slack', version: digestDef().version }, settings: { workspace: 'W1', project: 'P1' }, workflows: { 'completed-in-project': { enabled: true, settings: { channel: 'C1' } } } }); await inst.initialize(); await inst.onCreate(); - const name = 'data-defined-int9-wf_due_tomorrow_digest'; - expect(service._getSchedules()[name]).toMatchObject({ recurrence: { cron: '0 16 * * 1-5', timezone: 'America/New_York' }, state: 'ENABLED' }); - expect(inst.config.runtime.schedules[DIGEST]).toMatchObject({ jobId: name, event: DIGEST }); - - await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '3h' } } } } }); - expect(service._getSchedules()[name].recurrence).toEqual({ every: '3h' }); - - await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '10m' } } } } }); - expect(service._getSchedules()[name].recurrence).toEqual({ cron: '0 16 * * 1-5', timezone: 'America/New_York' }); - expect(inst.updateIntegrationMessages.execute).toHaveBeenCalledWith('int9', 'warnings', 'Schedule not changed', `${DIGEST}: Runs every 10 minutes at its most frequent; the minimum is 1h`, expect.any(Number)); - - await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: false } } } }); - expect(service._getSchedules()).toEqual({}); // switching the workflow off removes its schedule - - await inst.onUpdate({ config: { workflows: { 'due-tomorrow-digest': { enabled: true } } } }); + expect(inst.status).toBe('ENABLED'); + expect(await tickAt([inst], DUE)).toEqual([{ event: 'FRIGG_TRIGGER_RUN', data: { integrationId: 'int9', trigger: DIGEST, scheduledFor: '2026-10-19T20:00:00.000Z' } }]); + expect(await tickAt([inst], '2026-10-19T20:05:00Z')).toEqual([]); + + // Retimed to every 3 hours: due on the slots the installation's hash picks, 36 ticks apart. + const workflows = (digest) => ({ workflows: { 'completed-in-project': { enabled: true, settings: { channel: 'C1' } }, 'due-tomorrow-digest': digest } }); + await inst.onUpdate({ config: workflows({ enabled: true, schedule: { every: '3h' } }) }); + expect(inst.status).toBe('ENABLED'); + const slots = []; + for (let k = 0; k < 72; k += 1) if ((await tickAt([inst], new Date(Date.parse(DUE) + k * 300000).toISOString())).length) slots.push(k); + expect(slots).toHaveLength(2); + expect(slots[1] - slots[0]).toBe(36); + + // Under the floor: the definition's timing stays, and the installation is told why. + await inst.onUpdate({ config: workflows({ enabled: true, schedule: { every: '10m' } }) }); + expect((await tickAt([inst], DUE)).map((m) => m.data.trigger)).toEqual([DIGEST]); + expect(inst.updateIntegrationMessages.execute).toHaveBeenCalledWith('int9', 'warnings', 'Trigger timing not changed', `${DIGEST}: Runs every 10 minutes at its most frequent; the minimum is 1h`, expect.any(Number)); + + // Switched off: no trigger. Disabled or deleted: not read at all. + await inst.onUpdate({ config: workflows({ enabled: false }) }); + expect(await tickAt([inst], DUE)).toEqual([]); + await inst.onUpdate({ config: workflows({ enabled: true }) }); + expect(await tickAt([inst], DUE)).toHaveLength(1); + inst.status = 'DISABLED'; + expect(await tickAt([inst], DUE)).toEqual([]); await inst.onDelete(); - expect(service._getSchedules()).toEqual({}); - expect(inst.config.runtime.schedules).toEqual({}); + expect(spy).not.toHaveBeenCalled(); }); - it('an upgrade retimes schedules and keeps the install steps\' results beside them', async () => { + it('an upgrade changes what the next tick finds due, and drops a choice the new version disallows', async () => { const v2 = digestDef({ configurable: false, version: '2.0.0' }); v2.workflows.find((x) => x.id === 'due-tomorrow-digest').trigger.cron = '0 15 * * 1-5'; - const newInstall = setup([digestDef(), v2]); - const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version }, settings: { workspace: 'W1', project: 'P1' }, workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '2h' } }, 'completed-in-project': { enabled: true, settings: { channel: 'C1' } } } }); + const { install, tickAt } = setup([digestDef(), v2]); + const inst = install('int9', { definition: { id: 'asana-slack', version: digestDef().version }, settings: { workspace: 'W1', project: 'P1' }, workflows: { 'due-tomorrow-digest': { enabled: true, schedule: { every: '2h' } }, 'completed-in-project': { enabled: true, settings: { channel: 'C1' } } } }); await inst.initialize(); await inst.onCreate(); - expect(service._getSchedules()['data-defined-int9-wf_due_tomorrow_digest'].recurrence).toEqual({ every: '2h' }); + expect(inst.status).toBe('ENABLED'); + // every 2h (24 ticks): due only where this installation's hash puts it, never because of the 16:00 cron + const twoHourly = []; + for (let k = 0; k < 24; k += 1) if ((await tickAt([inst], new Date(Date.parse(DUE) + k * 300000).toISOString())).length) twoHourly.push(k); + expect(twoHourly).toHaveLength(1); const out = await inst.upgradeTo('2.0.0', { apply: true }); expect(out.applied).toBe(true); expect(out.actions).toContainEqual(expect.objectContaining({ kind: 'reschedule' })); - expect(service._getSchedules()['data-defined-int9-wf_due_tomorrow_digest'].recurrence).toEqual({ cron: '0 15 * * 1-5', timezone: 'America/New_York' }); - expect(inst.config.runtime.schedules[DIGEST].recurrence).toEqual({ cron: '0 15 * * 1-5', timezone: 'America/New_York' }); + expect(inst.config.workflows['due-tomorrow-digest'].schedule).toBeUndefined(); + expect(await tickAt([inst], '2026-10-19T19:00:00Z')).toEqual([expect.objectContaining({ data: expect.objectContaining({ trigger: DIGEST, scheduledFor: '2026-10-19T19:00:00.000Z' }) })]); + expect(await tickAt([inst], DUE)).toEqual([]); }); - it('a redelivered tick runs once: the tick\'s scheduled time is the dedupe id', async () => { - const newInstall = setup([digestDef()]); - const inst = newInstall({ definition: { id: 'asana-slack', version: digestDef().version } }); + it('a run delivered twice runs once: the trigger and its scheduled time are the dedupe id', async () => { + const { install } = setup([digestDef()]); + const inst = install('int9', { definition: { id: 'asana-slack', version: digestDef().version } }, 'ENABLED'); await inst.initialize(); const runs = []; inst.recordRun = async (r) => { runs.push(r); return r; }; - const tick = { event: DIGEST, data: { integrationId: 'int9', scheduleKey: DIGEST, tick: { id: 'exec-1', scheduledFor: '2026-10-19T20:00:00Z' } } }; - await inst.events[DIGEST].handler(tick); - await inst.events[DIGEST].handler(tick); + const run = { integrationId: 'int9', trigger: DIGEST, scheduledFor: '2026-10-19T20:00:00.000Z' }; + const dispatch = () => new IntegrationEventDispatcher(inst).dispatchJob({ event: 'FRIGG_TRIGGER_RUN', data: run }); + await dispatch(); + await dispatch(); expect(runs.map((r) => r.status)).toEqual([expect.not.stringMatching(/duplicate/), 'duplicate']); - expect(runs[0].trigger.eventId).toBe(`tick:${DIGEST}:2026-10-19T20:00:00Z`); - // An unfilled placeholder (a host that cannot supply the time) is not an id, so it cannot block every tick. - const unfilled = { data: { scheduleKey: DIGEST, tick: { scheduledFor: '{{frigg.tick.scheduledFor}}' } } }; - await inst.events[DIGEST].handler(unfilled); - await inst.events[DIGEST].handler(unfilled); - expect(runs.slice(2).map((r) => r.status)).not.toContain('duplicate'); + expect(runs[0].trigger.eventId).toBe(`tick:${DIGEST}:2026-10-19T20:00:00.000Z`); + // A different scheduled time is a different run. + await new IntegrationEventDispatcher(inst).dispatchJob({ event: 'FRIGG_TRIGGER_RUN', data: { ...run, scheduledFor: '2026-10-20T20:00:00.000Z' } }); + expect(runs[2].status).not.toBe('duplicate'); }); }); diff --git a/packages/core/integrations/data-defined/versioning.js b/packages/core/integrations/data-defined/versioning.js index 0ce3fe795..432562e53 100644 --- a/packages/core/integrations/data-defined/versioning.js +++ b/packages/core/integrations/data-defined/versioning.js @@ -173,4 +173,4 @@ function planUpgrade(installation, from, to) { module.exports = { diff, checkBump, planUpgrade, bumpOf }; const { validateRecurrence } = require('../../infrastructure/scheduler/recurrence'); -const { recurrenceOf } = require('../schedules'); +const { recurrenceOf } = require('../triggers'); diff --git a/packages/core/integrations/integration-base.js b/packages/core/integrations/integration-base.js index 9a293e1d5..24965d364 100644 --- a/packages/core/integrations/integration-base.js +++ b/packages/core/integrations/integration-base.js @@ -22,7 +22,7 @@ const { getTelemetry } = require('../telemetry/telemetry-runtime'); const { instrumentHandler } = require('../telemetry/instrument-handler'); const { bindTelemetryContext } = require('../telemetry/bind-telemetry-context'); const { getLogger } = require('../logs'); -const { desiredSchedules, entriesFromDefinition } = require('./schedules'); +const { TRIGGER_RUN, TRIGGER_TICK, resolveTrigger } = require('./triggers'); const constantsToBeMigrated = { defaultEvents: { @@ -36,6 +36,8 @@ const constantsToBeMigrated = { REFRESH_USER_ACTION_OPTIONS: 'REFRESH_USER_ACTION_OPTIONS', WEBHOOK_RECEIVED: 'WEBHOOK_RECEIVED', // HTTP handler, no DB ON_WEBHOOK: 'ON_WEBHOOK', // Queue worker, DB-connected + [TRIGGER_RUN]: TRIGGER_RUN, // Queue worker: one recurring trigger run (ADR-055) + [TRIGGER_TICK]: TRIGGER_TICK, // Queue worker: a trigger tick carrying on from its page cursor // etc... }, types: { @@ -92,6 +94,9 @@ class IntegrationBase { extensions: {}, // usage: { canonical: ['records.synced'], custom: { 'deals.enriched': { unit, label } } } usage: {}, + // Recurring triggers (ADR-055) — see integrations/triggers.js + // Shape: { [name]: { every | at | cron, timezone?, tick?, when?, overlap?, configurable?, handler } } + triggers: {}, display: { name: 'Integration Name', logo: '', @@ -172,6 +177,14 @@ class IntegrationBase { type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT, handler: this.onWebhook, }, + [TRIGGER_RUN]: { + type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT, + handler: this.onTriggerRun, + }, + [TRIGGER_TICK]: { + type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT, + handler: this.onTriggerTick, + }, }; } @@ -501,7 +514,6 @@ class IntegrationBase { async onCreate() { const needsConfig = await this.validateConfig(); await this.persistStatus(needsConfig ? 'NEEDS_CONFIG' : 'ENABLED'); - await this.syncSchedules(); } /** @@ -524,70 +536,84 @@ class IntegrationBase { } else if (this.status === 'NEEDS_CONFIG') { await this.persistStatus('ENABLED'); } - await this.syncSchedules(); + await this.reportTriggerChoices(); + } + + async onDelete(params) {} + + /** + * The recurring triggers declared for one installation (ADR-055): `Definition.triggers`, each with its `name`. + * Static so the tick can read them from a record without hydrating the installation; a class whose triggers + * depend on the installation (data-defined integrations, `Definition.instanceTriggers`) overrides it. + * @param {{ config?: Object, record?: Object }} [installation] + * @returns {Promise>} + */ + static async triggersFor(installation = {}) { + return Object.entries(this.Definition?.triggers || {}).map(([name, t]) => ({ ...t, name })); } - async onDelete(params) { - await this.syncSchedules({ remove: true }); + /** This installation's trigger declarations (the static `triggersFor`, unless a subclass knows better). */ + async triggerDeclarations() { + return this.constructor.triggersFor({ config: this.config || {}, record: this.record }); } /** - * The recurring schedules this installation should have (integrations/schedules.js). The default reads - * `Definition.schedules` and the installation's choices at `config.schedules`; data-defined integrations - * override it with their compiled definition. - * @returns {Array} schedule entries + * Run one recurring trigger: the queue worker dispatches `FRIGG_TRIGGER_RUN` here with + * `{ integrationId, trigger, scheduledFor }` after hydrating the installation. The handler is called as + * `handler(payload, ctx)` with `payload = { trigger, scheduledFor }` and `ctx = { modules, config, log, integration }` + * (ADR-054); a string handler names a method on this instance. Overlap and the run record: use-cases/run-trigger.js. */ - scheduleEntries() { - return entriesFromDefinition(this.constructor.Definition?.schedules, this.config || {}); + async onTriggerRun({ data = {} } = {}) { + const declarations = await this.triggerDeclarations(); + const declaration = declarations.find((t) => t.name === data.trigger); + const resolved = declaration ? resolveTrigger(declaration.name, declaration, this.config || {}) : null; + const payload = { trigger: data.trigger, scheduledFor: data.scheduledFor }; + const invoke = async (p) => { + const handler = typeof declaration.handler === 'string' ? this[declaration.handler] : declaration.handler; + if (typeof handler !== 'function') throw new Error(`Trigger "${declaration.name}" handler "${declaration.handler}" is not a method of ${this.constructor.Definition?.name}`); + return handler.call(this, p, { modules: this.modules, config: this.config || {}, log: this.logger, integration: this }); + }; + const { RunTrigger } = require('./use-cases/run-trigger'); + if (!this.triggerProcessRepository) { + const { createProcessRepository } = require('./repositories/process-repository-factory'); + this.triggerProcessRepository = createProcessRepository(); + } + const outcome = await new RunTrigger({ processRepository: this.triggerProcessRepository }).execute({ integration: this, resolved, invoke, payload }); + if (outcome.status !== 'ran') { + this.logger.info('Trigger run not started', { eventName: `${this.logger.name}.trigger_${outcome.status}`, trigger: data.trigger, reason: outcome.reason }); + } + return outcome; + } + + /** A tick that ran out of time continues here, on a dry instance, from its page cursor. */ + async onTriggerTick({ data = {}, context } = {}) { + const { runTriggerTick } = require('./use-cases/run-trigger-tick'); + return runTriggerTick(this.constructor, data, context); } /** - * Make the host's recurring schedules match `scheduleEntries()`, one per installation, and record what the host - * holds at `config.runtime.schedules`. Called by the default onCreate, onUpdate and onDelete; an override of - * those calls it itself. Does nothing for an integration that declares no schedules and has none recorded. - * - * Never throws: a schedule that could not be written is logged, reported in the result and left recorded as it - * was, and the next call retries it. A choice the definition does not allow keeps the default schedule and adds - * a warning message to the integration. - * - * @param {{ remove?: boolean }} [options] remove: delete every recorded schedule (uninstall) - * @returns {Promise} the reconcile outcome, or null when there was nothing to do + * Report each trigger timing this installation chose that cannot be used (not configurable, or under the + * floor): the trigger keeps its default, and the integration gets a warning message saying why. No-op for an + * integration without triggers. */ - async syncSchedules({ remove = false } = {}) { - const stored = this.config?.runtime?.schedules || {}; - const entries = remove ? [] : this.scheduleEntries() || []; - if (!entries.length && !Object.keys(stored).length) return null; - if (this.id === undefined || this.id === null) return null; - const name = this.constructor.Definition?.name; + async reportTriggerChoices() { + if (this.id === undefined || this.id === null) return []; try { - const { desired, problems } = desiredSchedules({ integrationName: name, integrationId: this.id, entries }); + const declarations = await this.triggerDeclarations(); + const problems = declarations + .map((t) => ({ name: t.name, problems: resolveTrigger(t.name, t, this.config || {}).problems })) + .filter((p) => p.problems.length); for (const p of problems) { - await this.updateIntegrationMessages.execute(this.id, 'warnings', 'Schedule not changed', `${p.key}: ${p.problems.join('; ')}`, Date.now()); - } - const commands = typeof this.schedulerCommands?.reconcileRecurringJobs === 'function' - ? this.schedulerCommands - : require('../application/commands/scheduler-commands').createSchedulerCommands({ integrationName: name }); - const outcome = await commands.reconcileRecurringJobs({ desired, stored, queueUrl: this.scheduleQueueUrl() }); - if (outcome.warning) return { ...outcome, problems }; - if (JSON.stringify(outcome.stored) !== JSON.stringify(stored)) { - await this.patchConfig({ runtime: { ...(this.config?.runtime || {}), schedules: outcome.stored } }); - } - if (outcome.failed.length) { - this.logger.error('Schedules not reconciled', { eventName: `${this.logger.name}.schedules_failed`, failed: outcome.failed }); + await this.updateIntegrationMessages.execute(this.id, 'warnings', 'Trigger timing not changed', `${p.name}: ${p.problems.join('; ')}`, Date.now()); } - return { ...outcome, problems }; + return problems; } catch (error) { - this.logger.error('Schedules not reconciled', { eventName: `${this.logger.name}.schedules_failed`, error }); - return { error: error.message }; + // A report, never a reason to fail the update. + this.logger.warn('Trigger choices not checked', { eventName: `${this.logger.name}.trigger_choices_unchecked`, error }); + return []; } } - /** The queue a schedule fires into: this integration's queue (`_QUEUE_URL`). */ - scheduleQueueUrl() { - if (typeof this.getQueueUrl === 'function') return this.getQueueUrl(); - return process.env[`${this.constructor.Definition.name.toUpperCase().replace(/-/g, '_')}_QUEUE_URL`]; - } - async getConfigOptions() { const options = { jsonSchema: {}, diff --git a/packages/core/integrations/repositories/integration-repository-documentdb.js b/packages/core/integrations/repositories/integration-repository-documentdb.js index 705ba2f19..b4e33e68e 100644 --- a/packages/core/integrations/repositories/integration-repository-documentdb.js +++ b/packages/core/integrations/repositories/integration-repository-documentdb.js @@ -28,7 +28,7 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface { return records.map((doc) => this._mapIntegration(doc)); } - async findIntegrations({ type, status } = {}) { + async findIntegrations({ type, status, limit, after } = {}) { const filter = {}; if (type) { filter['config.type'] = type; @@ -36,7 +36,11 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface { if (status) { filter.status = status; } - const records = await findMany(this.prisma, 'Integration', filter); + if (limit && after !== undefined && after !== null) { + // Extended JSON: $runCommandRaw does not match a BSON ObjectId instance inside an operator. + filter._id = { $gt: { $oid: fromObjectId(toObjectId(after)) } }; + } + const records = await findMany(this.prisma, 'Integration', filter, limit ? { sort: { _id: 1 }, limit, batchSize: limit } : {}); return records.map((doc) => this._mapIntegration(doc)); } diff --git a/packages/core/integrations/repositories/integration-repository-interface.js b/packages/core/integrations/repositories/integration-repository-interface.js index fb74db55d..66ee27358 100644 --- a/packages/core/integrations/repositories/integration-repository-interface.js +++ b/packages/core/integrations/repositories/integration-repository-interface.js @@ -28,6 +28,9 @@ class IntegrationRepositoryInterface { * @param {Object} [filter={}] * @param {string} [filter.type] - Integration type (config.type) * @param {string} [filter.status] - Integration status + * @param {number} [filter.limit] - Page size: at most this many rows, in id order + * @param {string|number} [filter.after] - With limit: only rows whose id sorts after this one (the last id of the + * previous page), so a caller can page through every row (the trigger tick does, ADR-055) * @returns {Promise} Array of integration objects (possibly empty) * @abstract */ diff --git a/packages/core/integrations/repositories/integration-repository-mongo.js b/packages/core/integrations/repositories/integration-repository-mongo.js index 6ec2e0eaf..d88528a95 100644 --- a/packages/core/integrations/repositories/integration-repository-mongo.js +++ b/packages/core/integrations/repositories/integration-repository-mongo.js @@ -61,20 +61,35 @@ class IntegrationRepositoryMongo extends IntegrationRepositoryInterface { * @param {string} [filter.status] - Integration status * @returns {Promise} Array of integration objects (possibly empty) */ - async findIntegrations({ type, status } = {}) { + async findIntegrations({ type, status, limit, after } = {}) { const where = {}; - if (type) { - where.config = { path: ['type'], equals: type }; - } if (status) { where.status = status; } + if (limit && after !== undefined && after !== null) { + where.id = { gt: after }; + } + if (type) { + // Prisma's MongoDB connector has no JSON path filter (`config: { path }` is refused), so the ids of the + // matching rows come from a raw find on config.type, then the rows load as usual. + const filter = { 'config.type': type }; + if (status) filter.status = status; + if (where.id) filter._id = { $gt: { $oid: String(after) } }; + const found = await this.prisma.integration.findRaw({ + filter, + options: { projection: { _id: 1 }, sort: { _id: 1 }, ...(limit ? { limit } : {}) }, + }); + const ids = found.map((doc) => doc._id?.$oid ?? String(doc._id)); + if (!ids.length) return []; + where.id = { in: ids }; + } const integrations = await this.prisma.integration.findMany({ where, include: { entities: true, }, + ...(limit ? { orderBy: { id: 'asc' }, take: limit } : {}), }); return integrations.map((integration) => ({ diff --git a/packages/core/integrations/repositories/integration-repository-paging.test.js b/packages/core/integrations/repositories/integration-repository-paging.test.js new file mode 100644 index 000000000..044a3703f --- /dev/null +++ b/packages/core/integrations/repositories/integration-repository-paging.test.js @@ -0,0 +1,112 @@ +/** + * The paged read the trigger tick uses (ADR-055 §3: "This needs a paged repository read"): + * findIntegrations({ type, status, limit, after }) against real databases, through every adapter, and a tick run + * over it end to end. + * + * Skipped unless a database URL is set (the data-defined persistence convention): + * - DATA_DEFINED_POSTGRES_URL a PostgreSQL database migrated with prisma-postgresql/migrations + * - DATA_DEFINED_MONGO_URL a MongoDB replica set with prisma-mongodb pushed (also runs the DocumentDB adapter) + */ +jest.mock('../../database/config', () => ({ DB_TYPE: 'mongodb', getDatabaseType: () => 'mongodb', PRISMA_LOG_LEVEL: 'error,warn' })); + +const { IntegrationRepositoryPostgres } = require('./integration-repository-postgres'); +const { IntegrationRepositoryMongo } = require('./integration-repository-mongo'); +const { IntegrationRepositoryDocumentDB } = require('./integration-repository-documentdb'); +const { RunTriggerTick } = require('../use-cases/run-trigger-tick'); + +const LEGS = [ + ['postgresql', process.env.DATA_DEFINED_POSTGRES_URL, () => require('../../generated/prisma-postgresql'), IntegrationRepositoryPostgres], + ['mongodb', process.env.DATA_DEFINED_MONGO_URL, () => require('../../generated/prisma-mongodb'), IntegrationRepositoryMongo], + ['documentdb (on mongodb)', process.env.DATA_DEFINED_MONGO_URL, () => require('../../generated/prisma-mongodb'), IntegrationRepositoryDocumentDB], +].filter(([, url]) => url); + +const describeLegs = LEGS.length ? describe.each(LEGS) : describe.skip.each([['no database URL set', null]]); + +describeLegs('findIntegrations paging on %s', (_, url, loadClient, Repo) => { + // A type of its own: other suites share the database. + const type = `paging-${process.pid}-${Date.now()}`; + let client, repo, ids, created; + + beforeAll(async () => { + const { PrismaClient } = loadClient(); + client = new PrismaClient({ datasourceUrl: url }); + repo = new Repo(); + repo.prisma = client; + ids = []; + created = []; + for (let i = 0; i < 12; i += 1) { + const row = await client.integration.create({ data: { config: { type, n: i }, status: i % 4 === 3 ? 'DISABLED' : 'ENABLED' } }); + created.push(row.id); + if (i % 4 !== 3) ids.push(String(row.id)); + } + // Rewrite the first row, so storage order (PostgreSQL appends the new tuple) is no longer id order. + await client.integration.update({ where: { id: created[0] }, data: { config: { type, n: 0, touched: true } } }); + }); + afterAll(async () => { + if (created?.length) await client.integration.deleteMany({ where: { id: { in: created } } }); + await client?.$disconnect(); + }); + + it('returns every ENABLED row of the type exactly once, in id order, a page at a time', async () => { + const seen = []; + let after; + const sizes = []; + for (;;) { + const page = await repo.findIntegrations({ type, status: 'ENABLED', limit: 4, after }); + sizes.push(page.length); + seen.push(...page.map((r) => String(r.id))); + if (page.length < 4) break; + after = page[page.length - 1].id; + } + expect(sizes).toEqual([4, 4, 1]); + expect(seen).toEqual(ids); + }); + + it('a page holds all of its rows, past the 101 a MongoDB first batch returns by default', async () => { + const bulkType = `${type}-bulk`; + await client.integration.createMany({ data: Array.from({ length: 110 }, (_, n) => ({ config: { type: bulkType, n }, status: 'ENABLED' })) }); + try { + const first = await repo.findIntegrations({ type: bulkType, status: 'ENABLED', limit: 105 }); + expect(first).toHaveLength(105); + const rest = await repo.findIntegrations({ type: bulkType, status: 'ENABLED', limit: 105, after: first[104].id }); + expect(rest).toHaveLength(5); + } finally { + const rows = await repo.findIntegrations({ type: bulkType }); + await client.integration.deleteMany({ where: { id: { in: rows.map((r) => (typeof created[0] === 'number' ? Number(r.id) : r.id)) } } }); + } + }); + + it('without limit it still returns everything (the existing callers)', async () => { + expect((await repo.findIntegrations({ type, status: 'ENABLED' })).map((r) => String(r.id)).sort()).toEqual([...ids].sort()); + }); + + it('a tick over the real read queues one run per ENABLED installation', async () => { + class Poller { static Definition = { name: type, triggers: { poll: { every: '5m', handler: 'poll' } } }; static async triggersFor() { return [{ name: 'poll', ...Poller.Definition.triggers.poll }]; } } + const sent = []; + const queue = { batchSend: async (entries) => { sent.push(...entries); }, send: async () => {} }; + const out = await new RunTriggerTick({ integrationClass: Poller, integrationRepository: repo, queue, queueUrl: 'q', pageSize: 4 }) + .execute({ tick: '5m', time: '2026-10-19T12:00:00Z' }); + expect(out).toMatchObject({ installations: 9, pages: 3, failed: [] }); + expect(sent.map((m) => String(m.data.integrationId))).toEqual(ids); + }); +}); + +describe('findIntegrations paging: the query each Prisma adapter sends', () => { + // Storage order usually equals id order in a test database, so ordering is pinned on the request itself. + const capture = (Repo, extra = {}) => { + const repo = new Repo(); + const calls = []; + repo.prisma = { integration: { findMany: async (args) => { calls.push(args); return []; }, findRaw: async () => [{ _id: { $oid: '6ad687411d7fe3ac5a4702a8' } }], ...extra } }; + return { repo, calls }; + }; + + it.each([['postgresql', IntegrationRepositoryPostgres, '7', 7], ['mongodb', IntegrationRepositoryMongo, '6ad687411d7fe3ac5a4702a7', '6ad687411d7fe3ac5a4702a7']])('%s: a limited read orders by id, takes the limit and starts after the cursor', async (_, Repo, after, expectedAfter) => { + const { repo, calls } = capture(Repo); + await repo.findIntegrations({ type: 't', status: 'ENABLED', limit: 500, after }); + expect(calls[0]).toMatchObject({ orderBy: { id: 'asc' }, take: 500 }); + if (Repo === IntegrationRepositoryPostgres) expect(calls[0].where.id).toEqual({ gt: expectedAfter }); + await repo.findIntegrations({ status: 'ENABLED' }); + expect(calls[1].orderBy).toBeUndefined(); + expect(calls[1].take).toBeUndefined(); + }); +}); diff --git a/packages/core/integrations/repositories/integration-repository-postgres.js b/packages/core/integrations/repositories/integration-repository-postgres.js index 3296bff0f..e7dbeb59e 100644 --- a/packages/core/integrations/repositories/integration-repository-postgres.js +++ b/packages/core/integrations/repositories/integration-repository-postgres.js @@ -99,7 +99,7 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface { * @param {string} [filter.status] - Integration status * @returns {Promise} Array of integration objects (possibly empty) */ - async findIntegrations({ type, status } = {}) { + async findIntegrations({ type, status, limit, after } = {}) { const where = {}; if (type) { where.config = { path: ['type'], equals: type }; @@ -107,12 +107,16 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface { if (status) { where.status = status; } + if (limit && after !== undefined && after !== null) { + where.id = { gt: this._convertId(after) }; + } const integrations = await this.prisma.integration.findMany({ where, include: { entities: true, }, + ...(limit ? { orderBy: { id: 'asc' }, take: limit } : {}), }); return integrations.map((integration) => { diff --git a/packages/core/integrations/schedules.js b/packages/core/integrations/schedules.js deleted file mode 100644 index d013e4ad0..000000000 --- a/packages/core/integrations/schedules.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Recurring schedules per installation: which ones an integration should have right now. Pure; no I/O. - * - * A developer declares them; an installation may change the timing of one only when the developer allowed it: - * - * static Definition = { - * schedules: { - * ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, // the user may pick any interval >= 15m - * DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, // fixed - * }, - * } - * - * The key is the event the schedule sends. An installation's choice lives at `config.schedules.` (`{ every }`, - * `{ cron, timezone }`, or `{ enabled: false }` when configurable); what the host actually holds is recorded by the - * runtime at `config.runtime.schedules.` (see scheduler-commands reconcileRecurringJobs). - * - * Data-defined integrations build the same entries from their compiled definition, so both forms share one - * reconcile path and one storage shape. - */ -const { createHash } = require('crypto'); -const { validateRecurrence } = require('../infrastructure/scheduler/recurrence'); - -const recurrenceOf = (r) => (r?.every !== undefined ? { every: r.every } : r?.cron !== undefined ? { cron: r.cron, ...(r.timezone ? { timezone: r.timezone } : {}) } : null); - -/** - * A stable host schedule name: `--`, within the 64 characters EventBridge allows (and only - * the characters it allows). Longer names keep a readable prefix and end in a short hash, so they stay unique. - */ -function scheduleJobId(integrationName, integrationId, key) { - const clean = (v) => String(v).replace(/[^0-9A-Za-z_.-]/g, '-'); - const full = `${clean(integrationName)}-${clean(integrationId)}-${clean(key).toLowerCase()}`; - if (full.length <= 64) return full; - const hash = createHash('sha256').update(full).digest('hex').slice(0, 10); - return `${full.slice(0, 53)}-${hash}`; -} - -/** - * @param {Object} params - * @param {string} params.integrationName - * @param {string|number} params.integrationId - * @param {Array<{key, event?, default: {every?|cron?, timezone?}, configurable?: false|{min?}, choice?: object, enabled?: boolean, payload?: object}>} params.entries - * default: the developer's recurrence. choice: the installation's override (honoured only when configurable). - * @returns {{ desired: Array<{key, jobId, event, recurrence, payload, enabled, source}>, problems: Array<{key, problems: string[]}> }} - * An entry whose choice is invalid keeps its default and is reported, so a bad edit never stops a schedule. - */ -function desiredSchedules({ integrationName, integrationId, entries = [] }) { - const desired = [], problems = []; - for (const e of entries) { - let recurrence = recurrenceOf(e.default), source = 'definition', enabled = e.enabled !== false; - const choice = e.choice && typeof e.choice === 'object' ? e.choice : null; - if (choice && (recurrenceOf(choice) || choice.enabled !== undefined)) { - if (!e.configurable) problems.push({ key: e.key, problems: ['This schedule is not configurable'] }); - else { - const picked = recurrenceOf(choice); - const issues = picked ? validateRecurrence(picked, { min: e.configurable.min }) : []; - if (issues.length) problems.push({ key: e.key, problems: issues }); - else { - if (picked) { recurrence = picked; source = 'installation'; } - if (choice.enabled === false) enabled = false; - } - } - } - const own = validateRecurrence(recurrence); - if (own.length) { problems.push({ key: e.key, problems: own.map((p) => `default: ${p}`) }); continue; } - desired.push({ - key: e.key, - jobId: scheduleJobId(integrationName, integrationId, e.key), - event: e.event || e.key, - recurrence, - payload: { integrationId, scheduleKey: e.key, ...(e.payload || {}) }, - enabled, - source, - }); - } - return { desired, problems }; -} - -/** The entries a hand-written integration declares: Definition.schedules plus the installation's choices. */ -function entriesFromDefinition(definitionSchedules = {}, config = {}) { - return Object.entries(definitionSchedules || {}).map(([key, s]) => ({ - key, - event: s.event || key, - default: s, - configurable: s.configurable ? (s.configurable === true ? {} : s.configurable) : false, - choice: config?.schedules?.[key], - })); -} - -module.exports = { desiredSchedules, entriesFromDefinition, scheduleJobId, recurrenceOf }; diff --git a/packages/core/integrations/tests/integration-base-schedules.test.js b/packages/core/integrations/tests/integration-base-schedules.test.js deleted file mode 100644 index fee77b3ed..000000000 --- a/packages/core/integrations/tests/integration-base-schedules.test.js +++ /dev/null @@ -1,137 +0,0 @@ -jest.mock('../../database/config', () => ({ - DB_TYPE: 'mongodb', - getDatabaseType: jest.fn(() => 'mongodb'), - PRISMA_LOG_LEVEL: 'error,warn', -})); - -const { IntegrationBase } = require('../integration-base'); -const { desiredSchedules, scheduleJobId } = require('../schedules'); -const { MockSchedulerAdapter } = require('../../infrastructure/scheduler/mock-scheduler-adapter'); -const { createSchedulerCommands } = require('../../application/commands/scheduler-commands'); - -const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/poller-queue'; - -describe('desiredSchedules', () => { - const entry = (over = {}) => ({ key: 'POLL', default: { every: '1h' }, configurable: { min: '15m' }, ...over }); - - it('uses the default, or the installation\'s choice where the definition allows it', () => { - expect(desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry()] }).desired).toEqual([ - { key: 'POLL', jobId: 'poller-i1-poll', event: 'POLL', recurrence: { every: '1h' }, payload: { integrationId: 'i1', scheduleKey: 'POLL' }, enabled: true, source: 'definition' }, - ]); - const chosen = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ choice: { cron: '0 9 * * 1-5', timezone: 'Europe/London' } })] }); - expect(chosen.desired[0]).toMatchObject({ recurrence: { cron: '0 9 * * 1-5', timezone: 'Europe/London' }, source: 'installation' }); - expect(chosen.problems).toEqual([]); - }); - - it('keeps the default and reports it when a choice breaks the floor or the schedule is fixed', () => { - const tooFast = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ choice: { every: '5m' } })] }); - expect(tooFast.desired[0]).toMatchObject({ recurrence: { every: '1h' }, source: 'definition' }); - expect(tooFast.problems).toEqual([{ key: 'POLL', problems: ['Runs every 5 minutes at its most frequent; the minimum is 15m'] }]); - const fixed = desiredSchedules({ integrationName: 'poller', integrationId: 'i1', entries: [entry({ configurable: false, choice: { every: '30m' } })] }); - expect(fixed.desired[0].recurrence).toEqual({ every: '1h' }); - expect(fixed.problems).toEqual([{ key: 'POLL', problems: ['This schedule is not configurable'] }]); - }); - - it('lets a configurable schedule be paused by the installation, and only then', () => { - expect(desiredSchedules({ integrationName: 'p', integrationId: 1, entries: [entry({ choice: { enabled: false } })] }).desired[0].enabled).toBe(false); - expect(desiredSchedules({ integrationName: 'p', integrationId: 1, entries: [entry({ configurable: false, choice: { enabled: false } })] }).desired[0].enabled).toBe(true); - }); - - it('builds host-safe names within 64 characters that stay unique', () => { - expect(scheduleJobId('hubspot', 42, 'ONGOING_SYNC')).toBe('hubspot-42-ongoing_sync'); - const long = scheduleJobId('data-defined', '65f1c0ffee0000000000abcd', 'SYNC_A_VERY_LONG_SYNC_IDENTIFIER_THAT_GOES_ON_ONGOING'); - const other = scheduleJobId('data-defined', '65f1c0ffee0000000000abcd', 'SYNC_A_VERY_LONG_SYNC_IDENTIFIER_THAT_GOES_ON_ONGOING_2'); - expect(long.length).toBeLessThanOrEqual(64); - expect(long).not.toBe(other); - expect(long).toMatch(/^[0-9A-Za-z_.-]+$/); - expect(scheduleJobId('a b', 'c/d', 'E:F')).toBe('a-b-c-d-e-f'); - }); -}); - -describe('IntegrationBase recurring schedules', () => { - let service, integration; - - class Poller extends IntegrationBase { - static Definition = { - name: 'poller', - version: '1.0.0', - modules: {}, - schedules: { - ONGOING_SYNC: { every: '1h', configurable: { min: '15m' } }, - DAILY_DIGEST: { cron: '0 8 * * 1-5', timezone: 'America/New_York' }, - }, - }; - } - - beforeEach(() => { - jest.spyOn(console, 'log').mockImplementation(() => {}); - process.env.POLLER_QUEUE_URL = QUEUE_URL; - service = new MockSchedulerAdapter(); - integration = new Poller(); - integration.id = 'int-1'; - integration.status = 'IN_CREATION'; - integration.config = {}; - integration.schedulerCommands = createSchedulerCommands({ integrationName: 'poller', schedulerService: service }); - integration.updateIntegrationStatus = { execute: jest.fn().mockResolvedValue(true) }; - integration.updateIntegrationMessages = { execute: jest.fn().mockResolvedValue(true) }; - integration.patchIntegrationConfig = { execute: jest.fn(async (id, patch) => ({ config: (integration.config = { ...integration.config, ...patch }) })) }; - integration.getConfigOptions = jest.fn().mockResolvedValue({ jsonSchema: {}, uiSchema: {} }); - }); - afterEach(() => { jest.restoreAllMocks(); delete process.env.POLLER_QUEUE_URL; }); - - it('onCreate creates one schedule per declared entry for this installation and records them on config', async () => { - await integration.onCreate(); - const schedules = service._getSchedules(); - expect(Object.keys(schedules).sort()).toEqual(['poller-int-1-daily_digest', 'poller-int-1-ongoing_sync']); - expect(schedules['poller-int-1-daily_digest'].recurrence).toEqual({ cron: '0 8 * * 1-5', timezone: 'America/New_York' }); - expect(service._simulateTrigger('poller-int-1-ongoing_sync', { id: 'x', scheduledFor: 't' })).toMatchObject({ event: 'ONGOING_SYNC', data: { integrationId: 'int-1', scheduleKey: 'ONGOING_SYNC' } }); - expect(integration.config.runtime.schedules).toEqual({ - ONGOING_SYNC: expect.objectContaining({ jobId: 'poller-int-1-ongoing_sync', recurrence: { every: '1h' }, event: 'ONGOING_SYNC' }), - DAILY_DIGEST: expect.objectContaining({ jobId: 'poller-int-1-daily_digest' }), - }); - }); - - it('onUpdate applies the installation\'s own interval, and refuses one below the floor with a warning', async () => { - await integration.onCreate(); - await integration.onUpdate({ config: { schedules: { ONGOING_SYNC: { every: '20m' } } } }); - expect(service._getSchedules()['poller-int-1-ongoing_sync'].recurrence).toEqual({ every: '20m' }); - - await integration.onUpdate({ config: { schedules: { ONGOING_SYNC: { every: '1m' }, DAILY_DIGEST: { cron: '0 * * * *' } } } }); - expect(service._getSchedules()['poller-int-1-ongoing_sync'].recurrence).toEqual({ every: '1h' }); // back to the default - expect(service._getSchedules()['poller-int-1-daily_digest'].recurrence.cron).toBe('0 8 * * 1-5'); // fixed - const warnings = integration.updateIntegrationMessages.execute.mock.calls.map((c) => c[3]); - expect(warnings).toEqual([ - 'ONGOING_SYNC: Runs every 1 minute at its most frequent; the minimum is 15m', - 'DAILY_DIGEST: This schedule is not configurable', - ]); - }); - - it('onDelete removes every recorded schedule, including one the code no longer declares', async () => { - await integration.onCreate(); - integration.config.runtime.schedules.LEGACY = { jobId: 'poller-int-1-legacy', recurrence: { every: '1d' }, event: 'LEGACY' }; - await service.upsertRecurring({ scheduleName: 'poller-int-1-legacy', recurrence: { every: '1d' }, queueResourceId: 'arn:q', payload: {} }); - await integration.onDelete(); - expect(service._getSchedules()).toEqual({}); - expect(integration.config.runtime.schedules).toEqual({}); - }); - - it('a schedule the host refuses stays unrecorded and is logged; the lifecycle carries on', async () => { - jest.spyOn(service, 'upsertRecurring').mockRejectedValue(new Error('AccessDenied')); - const error = jest.spyOn(integration.logger, 'error').mockImplementation(() => {}); - await integration.onCreate(); - expect(integration.updateIntegrationStatus.execute).toHaveBeenCalledWith('int-1', 'ENABLED'); - expect(integration.config.runtime).toBeUndefined(); - expect(error).toHaveBeenCalledWith('Schedules not reconciled', expect.objectContaining({ eventName: expect.stringMatching(/schedules_failed$/) })); - }); - - it('does nothing at all for an integration with no schedules', async () => { - class Plain extends IntegrationBase { static Definition = { name: 'plain', version: '1.0.0', modules: {} }; } - const plain = new Plain(); - plain.id = 'p1'; - plain.config = {}; - plain.patchIntegrationConfig = { execute: jest.fn() }; - expect(await plain.syncSchedules()).toBeNull(); - expect(await plain.syncSchedules({ remove: true })).toBeNull(); - expect(plain.patchIntegrationConfig.execute).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/core/integrations/tests/trigger-runtime.test.js b/packages/core/integrations/tests/trigger-runtime.test.js new file mode 100644 index 000000000..df0c36870 --- /dev/null +++ b/packages/core/integrations/tests/trigger-runtime.test.js @@ -0,0 +1,271 @@ +/** + * The trigger runtime (ADR-055 §3): the tick that pages ENABLED installations and queues the runs that are due, and + * the run on the queue worker (IntegrationBase.onTriggerRun with the RunTrigger use case). Persistence and SQS are + * fakes that record what was asked of them; the due rules are the real ones. + */ +jest.mock('../../database/config', () => ({ + DB_TYPE: 'mongodb', + getDatabaseType: jest.fn(() => 'mongodb'), + PRISMA_LOG_LEVEL: 'error,warn', +})); + +const { IntegrationBase } = require('../integration-base'); +const { RunTriggerTick } = require('../use-cases/run-trigger-tick'); +const { RunTrigger } = require('../use-cases/run-trigger'); +const { IntegrationEventDispatcher } = require('../../handlers/integration-event-dispatcher'); +const { hashId } = require('../triggers'); + +const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/poller-queue'; +const at = (iso) => Date.parse(iso); + +/** findIntegrations over an array, honouring type, status, limit and after the way the adapters do. */ +function fakeRepository(rows) { + const calls = []; + return { + calls, + async findIntegrations({ type, status, limit, after } = {}) { + calls.push({ type, status, limit, after }); + const sorted = rows.filter((r) => r.config.type === type && r.status === status).sort((a, b) => a.id - b.id); + const rest = after === undefined ? sorted : sorted.filter((r) => r.id > after); + return limit ? rest.slice(0, limit) : rest; + }, + }; +} + +function fakeQueue() { + const q = { batches: [], sent: [] }; + q.batchSend = async (entries, queueUrl, options = {}) => { + q.batches.push({ queueUrl, entries: entries.map((e, i) => ({ ...e, delay: options.delaySeconds ? options.delaySeconds(e, i) : 0 })) }); + }; + q.send = async (message, queueUrl) => { q.sent.push({ message, queueUrl }); }; + q.runs = () => q.batches.flatMap((b) => b.entries); + return q; +} + +class Poller extends IntegrationBase { + static Definition = { + name: 'poller', + modules: {}, + triggers: { + poll: { every: '15m', handler: 'poll' }, + sync: { every: '1h', configurable: { min: '15m' }, when: (c) => c.syncOn !== false, handler: 'sync' }, + digest: { cron: '0 8 * * 1-5', timezone: 'America/New_York', tick: '15m', handler: 'digest' }, + nightly: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler: 'nightly' }, + }, + }; +} + +const row = (id, config = {}, status = 'ENABLED') => ({ id, status, config: { type: 'poller', ...config } }); + +describe('RunTriggerTick', () => { + const tick = (rows, opts = {}) => { + const repository = fakeRepository(rows); + const queue = fakeQueue(); + const useCase = new RunTriggerTick({ integrationClass: Poller, integrationRepository: repository, queue, queueUrl: QUEUE_URL, pageSize: opts.pageSize || 500 }); + return { repository, queue, useCase }; + }; + + it('reads only ENABLED installations of this integration and queues FRIGG_TRIGGER_RUN for each due one', async () => { + const rows = [row(1), row(2), row(3, {}, 'DISABLED'), { id: 4, status: 'ENABLED', config: { type: 'other' } }]; + const { repository, queue, useCase } = tick(rows); + // Every 15m trigger is due at every 15m slot (cadence 1); sync (1h, cadence 4) only where the hash says. + const out = await useCase.execute({ tick: '15m', time: '2026-10-19T12:00:07Z' }); + expect(repository.calls[0]).toEqual({ type: 'poller', status: 'ENABLED', limit: 500, after: undefined }); + const polls = queue.runs().filter((m) => m.data.trigger === 'poll'); + expect(polls.map((m) => m.data.integrationId)).toEqual([1, 2]); + expect(polls[0]).toMatchObject({ event: 'FRIGG_TRIGGER_RUN', data: { integrationId: 1, trigger: 'poll', scheduledFor: '2026-10-19T12:00:00.000Z' } }); + expect(queue.batches.every((b) => b.queueUrl === QUEUE_URL)).toBe(true); + expect(out).toMatchObject({ installations: 2, pages: 1, continued: false, failed: [] }); + // The digest fired at 08:00 New York (12:00Z), inside (11:45, 12:00]; nightly is not on this tick. + expect(queue.runs().filter((m) => m.data.trigger === 'digest').map((m) => m.data.scheduledFor)).toEqual(['2026-10-19T12:00:00.000Z', '2026-10-19T12:00:00.000Z']); + expect(queue.runs().some((m) => m.data.trigger === 'nightly')).toBe(false); + }); + + it('spreads a 1h trigger over the hour: over four 15m ticks each installation runs once, in its hash\'s slot', async () => { + const rows = Array.from({ length: 40 }, (_, i) => row(i + 1)); + const { queue, useCase } = tick(rows); + for (const t of ['12:00', '12:15', '12:30', '12:45']) await useCase.execute({ tick: '15m', time: `2026-10-19T${t}:00Z` }); + const syncs = queue.runs().filter((m) => m.data.trigger === 'sync'); + expect(syncs).toHaveLength(40); + expect(new Set(syncs.map((m) => m.data.integrationId)).size).toBe(40); + expect(new Set(syncs.map((m) => m.data.scheduledFor)).size).toBe(4); + }); + + it('honours each installation\'s chosen timing and `when`, and never a choice under the floor', async () => { + const rows = [row(1, { triggers: { sync: { every: '15m' } } }), row(2, { syncOn: false }), row(3, { triggers: { sync: { every: '5m' } } })]; + const { queue, useCase } = tick(rows); + for (const t of ['12:00', '12:15', '12:30', '12:45']) await useCase.execute({ tick: '15m', time: `2026-10-19T${t}:00Z` }); + const syncsOf = (id) => queue.runs().filter((m) => m.data.trigger === 'sync' && m.data.integrationId === id).length; + expect(syncsOf(1)).toBe(4); // chose every 15m + expect(syncsOf(2)).toBe(0); // `when` excludes it + expect(syncsOf(3)).toBe(1); // 5m is under the floor: the default hourly runs + }); + + it('decides the same for the same schedule time, wherever in the slot the tick runs', async () => { + const rows = Array.from({ length: 20 }, (_, i) => row(i + 1)); + const a = tick(rows); const b = tick(rows); + await a.useCase.execute({ tick: '15m', time: '2026-10-19T12:15:00Z' }); + await b.useCase.execute({ tick: '15m', time: '2026-10-19T12:29:59Z' }); + expect(b.queue.runs()).toEqual(a.queue.runs()); + }); + + it('a tick runs only the triggers on that tick, so a 15m trigger is not run again by the 5m schedule', async () => { + class Mixed extends Poller { static Definition = { ...Poller.Definition, triggers: { fast: { every: '5m', handler: 'poll' }, poll: { every: '15m', handler: 'poll' } } }; } + const queue = fakeQueue(); + const useCase = () => new RunTriggerTick({ integrationClass: Mixed, integrationRepository: fakeRepository([row(1)]), queue, queueUrl: QUEUE_URL }); + await useCase().execute({ tick: '15m', time: '2026-10-19T12:00:00Z' }); + for (const t of ['12:00', '12:05', '12:10']) await useCase().execute({ tick: '5m', time: `2026-10-19T${t}:00Z` }); + expect(queue.runs().map((m) => m.data.trigger)).toEqual(['poll', 'fast', 'fast', 'fast']); + }); + + it('jitters runs within the tick by a stable delay, at most min(tick, 900 s)', async () => { + const { queue, useCase } = tick([row(1), row(2)]); + await useCase.execute({ tick: '15m', time: '2026-10-19T12:00:00Z' }); + for (const m of queue.runs()) { + expect(m.delay).toBe(hashId(`${m.data.integrationId}:${m.data.trigger}`) % 900); + expect(m.delay).toBeLessThan(900); + } + }); + + it('an at trigger runs for every ENABLED installation that passes `when`, without jitter', async () => { + const { queue, useCase } = tick([row(1), row(2), row(3, {}, 'ERROR')]); + await useCase.execute({ at: 'nightly', time: '2026-10-19T02:00:13Z' }); + expect(queue.runs()).toEqual([1, 2].map((id) => ({ event: 'FRIGG_TRIGGER_RUN', data: { integrationId: id, trigger: 'nightly', scheduledFor: '2026-10-19T02:00:00.000Z' }, delay: 0 }))); + }); + + it('pages by id until a short page, and hands its cursor to the queue when the deadline nears', async () => { + const rows = Array.from({ length: 7 }, (_, i) => row(i + 1)); + const paged = tick(rows, { pageSize: 3 }); + const out = await paged.useCase.execute({ tick: '15m', time: '2026-10-19T12:00:00Z' }); + expect(paged.repository.calls.map((c) => c.after)).toEqual([undefined, 3, 6]); + expect(out).toMatchObject({ installations: 7, pages: 3, continued: false }); + + const hurried = tick(rows, { pageSize: 3 }); + const cut = await hurried.useCase.execute({ tick: '15m', time: '2026-10-19T12:00:00Z', remainingMs: () => 5000 }); + expect(cut).toMatchObject({ installations: 3, pages: 1, continued: true }); + expect(hurried.queue.sent).toEqual([{ queueUrl: QUEUE_URL, message: { event: 'FRIGG_TRIGGER_TICK', data: { tick: '15m', time: '2026-10-19T12:00:00.000Z', cursor: 3 } } }]); + // The continuation decides the rest of the same slot. + await hurried.useCase.execute({ ...hurried.queue.sent[0].message.data }); + expect(hurried.queue.runs()).toEqual(paged.queue.runs()); + }); + + it('one installation\'s bad config does not stop the others', async () => { + class Picky extends Poller { + static Definition = { ...Poller.Definition, name: 'poller', triggers: { poll: { every: '15m', when: (c) => { if (c.broken) throw new Error('bad config'); return true; }, handler: 'poll' } } }; + } + const repository = fakeRepository([row(1, { broken: true }), row(2)]); + const queue = fakeQueue(); + const out = await new RunTriggerTick({ integrationClass: Picky, integrationRepository: repository, queue, queueUrl: QUEUE_URL }).execute({ tick: '15m', time: '2026-10-19T12:00:00Z' }); + expect(queue.runs().map((m) => m.data.integrationId)).toEqual([2]); + expect(out.failed).toEqual([{ integrationId: 1, error: 'bad config' }]); + }); + + it('refuses to run without a queue or with a tick that is not a duration', async () => { + const repository = fakeRepository([]); + await expect(new RunTriggerTick({ integrationClass: Poller, integrationRepository: repository, queue: fakeQueue() }).execute({ tick: '15m' })).rejects.toThrow(/No queue URL/); + await expect(new RunTriggerTick({ integrationClass: Poller, integrationRepository: repository, queue: fakeQueue(), queueUrl: QUEUE_URL }).execute({ tick: 'soon' })).rejects.toThrow(/not a duration/); + }); +}); + +/** A process repository over an array. */ +function fakeProcesses(initial = []) { + const rows = initial.map((r) => ({ ...r })); + let next = 100; + return { + rows, + async findByIntegrationAndType(integrationId, type) { return rows.filter((r) => r.integrationId === integrationId && r.type === type); }, + async create(data) { const r = { id: String(next++), ...data, updatedAt: new Date().toISOString() }; rows.push(r); return r; }, + async update(id, updates) { const r = rows.find((x) => x.id === id); Object.assign(r, updates, { updatedAt: new Date().toISOString() }); return r; }, + }; +} + +describe('the run: FRIGG_TRIGGER_RUN on the queue worker', () => { + const make = ({ status = 'ENABLED', config = {}, processes = fakeProcesses() } = {}) => { + const calls = []; + class Running extends Poller { + async poll(payload, ctx) { calls.push({ method: 'poll', self: this, payload, ctx }); return 'polled'; } + async sync(payload) { calls.push({ method: 'sync', payload }); throw Object.assign(new Error('upstream 503'), { statusCode: 503 }); } + } + const inst = new Running({ id: 'int1', userId: 'u1', status, config: { type: 'poller', ...config } }); + inst.triggerProcessRepository = processes; + inst.updateIntegrationMessages = { execute: jest.fn(async () => true) }; + return { inst, calls, processes, dispatch: (data) => new IntegrationEventDispatcher(inst).dispatchJob({ event: 'FRIGG_TRIGGER_RUN', data }) }; + }; + + it('runs the handler as handler(payload, ctx) with the instance as this, and records the run', async () => { + const { inst, calls, processes, dispatch } = make(); + const out = await dispatch({ integrationId: 'int1', trigger: 'poll', scheduledFor: '2026-10-19T12:00:00.000Z' }); + expect(out).toEqual({ status: 'ran', result: 'polled' }); + expect(calls[0].self).toBe(inst); + expect(calls[0].payload).toEqual({ trigger: 'poll', scheduledFor: '2026-10-19T12:00:00.000Z' }); + expect(Object.keys(calls[0].ctx).sort()).toEqual(['config', 'integration', 'log', 'modules']); + expect(processes.rows).toEqual([expect.objectContaining({ integrationId: 'int1', userId: 'u1', type: 'TRIGGER:poll', state: 'COMPLETED' })]); + }); + + it('discards the run for an installation that is not ENABLED, or a trigger it no longer has', async () => { + for (const status of ['DISABLED', 'NEEDS_CONFIG', 'ERROR']) { + const { calls, dispatch } = make({ status }); + expect(await dispatch({ trigger: 'poll', scheduledFor: 'x' })).toMatchObject({ status: 'discarded' }); + expect(calls).toEqual([]); + } + const gated = make({ config: { syncOn: false } }); + expect(await gated.dispatch({ trigger: 'sync', scheduledFor: 'x' })).toEqual({ status: 'discarded', reason: 'trigger is when' }); + expect(await gated.dispatch({ trigger: 'gone', scheduledFor: 'x' })).toEqual({ status: 'discarded', reason: 'trigger is not declared' }); + expect(gated.calls).toEqual([]); + }); + + it('overlap skip: a run still going (touched within 2 x tick) makes the next one skip; a stale one does not', async () => { + const fresh = make({ processes: fakeProcesses([{ id: '1', integrationId: 'int1', type: 'TRIGGER:poll', state: 'RUNNING', updatedAt: new Date(Date.now() - 20 * 60000).toISOString() }]) }); + expect(await fresh.dispatch({ trigger: 'poll', scheduledFor: 'x' })).toMatchObject({ status: 'skipped' }); + expect(fresh.calls).toEqual([]); + const stale = make({ processes: fakeProcesses([{ id: '1', integrationId: 'int1', type: 'TRIGGER:poll', state: 'RUNNING', updatedAt: new Date(Date.now() - 31 * 60000).toISOString() }]) }); + expect(await stale.dispatch({ trigger: 'poll', scheduledFor: 'x' })).toMatchObject({ status: 'ran' }); + expect(stale.processes.rows).toHaveLength(1); // the record is reused, not one per run + expect(stale.processes.rows[0].state).toBe('COMPLETED'); + }); + + it('a failed run is marked ERROR and rethrown to the worker, so its retry is not skipped by itself', async () => { + const { dispatch, processes, calls } = make(); + await expect(dispatch({ trigger: 'sync', scheduledFor: 'x' })).rejects.toMatchObject({ statusCode: 503 }); + expect(processes.rows[0]).toMatchObject({ type: 'TRIGGER:sync', state: 'ERROR', results: expect.objectContaining({ error: 'upstream 503' }) }); + await expect(dispatch({ trigger: 'sync', scheduledFor: 'x' })).rejects.toMatchObject({ statusCode: 503 }); + expect(calls.filter((c) => c.method === 'sync')).toHaveLength(2); + }); + + it('overlap allow keeps no record', async () => { + const out = await new RunTrigger({ processRepository: { findByIntegrationAndType: () => { throw new Error('not read'); } } }) + .execute({ integration: { id: 'i', status: 'ENABLED' }, resolved: { active: true, overlap: 'allow' }, invoke: async () => 1, payload: { trigger: 't' } }); + expect(out).toEqual({ status: 'ran', result: 1 }); + }); + + it('onUpdate warns about a chosen timing it cannot use, and touches no scheduler', async () => { + const { inst } = make({ config: { triggers: { sync: { every: '5m' } } } }); + inst.persistStatus = jest.fn(async () => {}); + inst.validateConfig = async () => false; + await inst.onUpdate({}); + expect(inst.updateIntegrationMessages.execute).toHaveBeenCalledWith('int1', 'warnings', 'Trigger timing not changed', 'sync: Runs every 5 minutes at its most frequent; the minimum is 15m', expect.any(Number)); + }); + + it('a continuation tick runs on a dry instance', async () => { + const spy = jest.spyOn(require('../use-cases/run-trigger-tick'), 'runTriggerTick').mockResolvedValue({ continued: false }); + const dry = new Poller(); + await new IntegrationEventDispatcher(dry).dispatchJob({ event: 'FRIGG_TRIGGER_TICK', data: { tick: '15m', time: 't', cursor: 3 } }); + expect(spy).toHaveBeenCalledWith(Poller, { tick: '15m', time: 't', cursor: 3 }, undefined); + spy.mockRestore(); + }); +}); + +describe('lifecycle needs no trigger work', () => { + it('create, update and delete never reach the scheduler', async () => { + const scheduler = require('../../application/commands/scheduler-commands'); + const spy = jest.spyOn(scheduler, 'createSchedulerCommands'); + const inst = new Poller({ id: 'int1', status: 'IN_CREATION', config: { type: 'poller' } }); + inst.persistStatus = jest.fn(async () => {}); + inst.updateIntegrationMessages = { execute: jest.fn() }; + await inst.onCreate(); + await inst.onUpdate({}); + await inst.onDelete(); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); +}); diff --git a/packages/core/integrations/tests/triggers.test.js b/packages/core/integrations/tests/triggers.test.js new file mode 100644 index 000000000..f835c2870 --- /dev/null +++ b/packages/core/integrations/tests/triggers.test.js @@ -0,0 +1,169 @@ +/** + * Recurring triggers (ADR-055 + its ADR-063 amendment): declarations, what devtools provisions, how one installation + * resolves a trigger (config, `when`, a chosen timing within its floor) and the stateless due check. + */ +const { + validateTriggers, + provisioningOf, + hasTriggers, + tickMinutesOf, + resolveTrigger, + dueRun, + hashId, + slotStart, + durationOf, +} = require('../triggers'); + +const MIN = 60000; +const handler = 'run'; + +describe('declarations', () => { + it('accepts the ADR-055 shapes and the amendment\'s configurable / cron forms', () => { + expect(validateTriggers({ + poll: { every: '15m', handler }, + perPlan: { every: (c) => c.pollEvery, when: (c) => c.on, overlap: 'allow', handler: async () => {} }, + nightly: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler }, + digest: { cron: '0 8 * * 1-5', timezone: 'America/New_York', configurable: { min: '1h' }, handler }, + sync: { every: '1h', configurable: true, tick: '5m', handler }, + })).toEqual([]); + }); + + it('refuses what no tick can run, and says which trigger', () => { + expect(validateTriggers({ + none: { handler }, + both: { every: '1h', at: 'cron(0 3 * * ? *)', handler }, + noHandler: { every: '1h' }, + badOverlap: { every: '1h', overlap: 'queue', handler }, + tooLong: { every: '2d', handler }, + badAt: { at: '0 3 * * *', handler }, + atConfigurable: { at: 'cron(0 3 * * ? *)', configurable: true, handler }, + badZone: { cron: '0 8 * * *', timezone: 'Mars/Base', handler }, + ownFloor: { every: '10m', configurable: { min: '15m' }, handler }, + badTick: { every: '1h', tick: '30s', handler }, + })).toEqual([ + 'none: Give exactly one of every, at or cron', + 'both: Give exactly one of every, at or cron', + 'noHandler: handler must be a function or a method name', + 'badOverlap: overlap must be skip or allow', + 'tooLong: every "2d" is longer than 1d; use at or cron', + 'badAt: at "0 3 * * *" must be an EventBridge cron(...) expression with six fields', + 'atConfigurable: An at trigger runs at one time for every installation; use cron to let installations choose', + 'badZone: Unknown time zone "Mars/Base"', + 'ownFloor: The default breaks its own minimum: Runs every 10 minutes at its most frequent; the minimum is 15m', + 'badTick: tick "30s" is not a duration between 1m and 1d', + ]); + }); + + it('ticks: tick when set, a constant every, 5m for a function, the floor when configurable, 5m or less for cron', () => { + expect(tickMinutesOf({ every: '1h', tick: '10m' })).toBe(10); + expect(tickMinutesOf({ every: '1h' })).toBe(60); + expect(tickMinutesOf({ every: () => '1h' })).toBe(5); + expect(tickMinutesOf({ every: '1h', configurable: { min: '15m' } })).toBe(15); + expect(tickMinutesOf({ every: '1h', configurable: true })).toBe(5); + expect(tickMinutesOf({ cron: '0 8 * * *' })).toBe(5); + expect(tickMinutesOf({ cron: '*/2 * * * *' })).toBe(2); + expect(tickMinutesOf({ at: 'cron(0 3 * * ? *)' })).toBeNull(); + }); + + it('provisions one entry per distinct tick and one per at trigger, from Definition.triggers and instanceTriggers', () => { + class Code { static Definition = { name: 'code', triggers: { + a: { every: '15m', handler }, b: { every: '1h', configurable: { min: '15m' }, handler }, + c: { cron: '0 8 * * 1-5', timezone: 'America/New_York', handler }, d: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler }, + } }; } + expect(provisioningOf(Code)).toEqual({ ticks: [5, 15], at: [{ name: 'd', at: 'cron(0 3 * * ? *)', timezone: 'Europe/London' }] }); + class Defined { static Definition = { name: 'defined', instanceTriggers: { tick: '5m' } }; } + expect(provisioningOf(Defined)).toEqual({ ticks: [5], at: [] }); + expect(hasTriggers(Defined)).toBe(true); + expect(hasTriggers(class { static Definition = { name: 'none', triggers: {} }; })).toBe(false); + class Bad { static Definition = { name: 'bad', triggers: { x: { every: 'often', handler } } }; } + expect(() => provisioningOf(Bad)).toThrow(/Integration "bad" declares invalid triggers: x: every "often"/); + expect([1, 5, 60, 90, 1440].map(durationOf)).toEqual(['1m', '5m', '1h', '90m', '1d']); + }); +}); + +describe('one installation', () => { + const sync = { every: '1h', configurable: { min: '15m' }, handler }; + + it('uses the default, or the installation\'s choice (config.triggers.) where the floor allows it', () => { + expect(resolveTrigger('sync', sync, {})).toMatchObject({ recurrence: { every: '1h' }, tickMinutes: 15, cadence: 4, source: 'definition', active: true, problems: [] }); + expect(resolveTrigger('sync', sync, { triggers: { sync: { every: '30m' } } })).toMatchObject({ recurrence: { every: '30m' }, cadence: 2, source: 'installation' }); + // A cadence rounds up to whole ticks: 20m on a 15m tick runs every 2 ticks (30m). + expect(resolveTrigger('sync', sync, { triggers: { sync: { every: '20m' } } }).cadence).toBe(2); + expect(resolveTrigger('sync', sync, { triggers: { sync: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' } } })).toMatchObject({ recurrence: { cron: '0 9 * * 1-5', timezone: 'Europe/Paris' }, source: 'installation' }); + }); + + it('keeps the default and reports it when a choice breaks the floor, or the trigger is fixed', () => { + expect(resolveTrigger('sync', sync, { triggers: { sync: { every: '5m' } } })).toMatchObject({ + recurrence: { every: '1h' }, source: 'definition', active: true, problems: ['Runs every 5 minutes at its most frequent; the minimum is 15m'], + }); + expect(resolveTrigger('sync', sync, { triggers: { sync: { cron: '*/5 * * * *' } } }).problems).toEqual(['Runs every 5 minutes at its most frequent; the minimum is 15m']); + expect(resolveTrigger('sync', sync, { triggers: { sync: { every: '3d' } } })).toMatchObject({ recurrence: { every: '1h' }, problems: ['every "3d" is longer than 1d'] }); + const fixed = { every: '1h', handler }; + expect(resolveTrigger('fixed', fixed, { triggers: { fixed: { every: '30m' } } })).toMatchObject({ recurrence: { every: '1h' }, problems: ['This trigger is not configurable'] }); + }); + + it('a choice handed in by the caller (data-defined: workflows..schedule) wins over config.triggers', () => { + expect(resolveTrigger('wf', { ...sync, choice: { every: '2h' } }, { triggers: { wf: { every: '30m' } } }).recurrence).toEqual({ every: '2h' }); + }); + + it('is inactive when `when` excludes the installation, or it paused a configurable trigger', () => { + const gated = { every: '15m', when: (c) => c.direction !== 'crm->erp', handler }; + expect(resolveTrigger('g', gated, { direction: 'crm->erp' })).toMatchObject({ active: false, inactive: 'when' }); + expect(resolveTrigger('g', gated, { direction: 'two-way' }).active).toBe(true); + expect(resolveTrigger('sync', sync, { triggers: { sync: { enabled: false } } })).toMatchObject({ active: false, inactive: 'paused' }); + expect(resolveTrigger('f', { every: '1h', handler }, { triggers: { f: { enabled: false } } })).toMatchObject({ active: true, problems: ['This trigger is not configurable'] }); + }); + + it('reads every from the config when it is a function, and an invalid answer stops only that trigger', () => { + const perPlan = { every: (c) => c.pollEvery, handler }; + expect(resolveTrigger('p', perPlan, { pollEvery: '15m' })).toMatchObject({ tickMinutes: 5, cadence: 3, active: true }); + expect(resolveTrigger('p', perPlan, { pollEvery: 'sometimes' })).toMatchObject({ active: false, inactive: 'invalid' }); + }); +}); + +describe('the due check', () => { + const at = (iso) => Date.parse(iso); + + it('every: each installation is due exactly once per cadence, spread by its id', () => { + const r = resolveTrigger('poll', { every: '1h', tick: '15m', handler }, {}); + expect(r.cadence).toBe(4); + const ids = Array.from({ length: 200 }, (_, i) => `int-${i}`); + const base = at('2026-10-19T12:00:00Z'); + const dueAt = Object.fromEntries(ids.map((id) => [id, []])); + for (let k = 0; k < 8; k += 1) { + for (const id of ids) if (dueRun(r, { integrationId: id, slotMs: base + k * 15 * MIN })) dueAt[id].push(k); + } + for (const id of ids) { + expect(dueAt[id]).toHaveLength(2); // two cadences in eight ticks + expect(dueAt[id][1] - dueAt[id][0]).toBe(4); // exactly one cadence apart + } + // and spread: every one of the four slots of a cadence gets some installations + expect(new Set(ids.map((id) => dueAt[id][0])).size).toBe(4); + const first = ids.find((id) => dueAt[id][0] === 0); + expect(dueRun(r, { integrationId: first, slotMs: base })).toEqual({ scheduledFor: '2026-10-19T12:00:00.000Z' }); + }); + + it('cron: due on the tick whose window holds a firing, with that firing as scheduledFor', () => { + const r = resolveTrigger('digest', { cron: '0 8 * * 1-5', timezone: 'America/New_York', tick: '5m', handler }, {}); + expect(dueRun(r, { integrationId: 'a', slotMs: at('2026-10-19T12:00:00Z') })).toEqual({ scheduledFor: '2026-10-19T12:00:00.000Z' }); + expect(dueRun(r, { integrationId: 'a', slotMs: at('2026-10-19T12:05:00Z') })).toBeNull(); + expect(dueRun(r, { integrationId: 'a', slotMs: at('2026-10-19T11:55:00Z') })).toBeNull(); + const late = resolveTrigger('d', { cron: '3 8 * * 1-5', timezone: 'America/New_York', tick: '5m', handler }, {}); + expect(dueRun(late, { integrationId: 'a', slotMs: at('2026-10-19T12:05:00Z') })).toEqual({ scheduledFor: '2026-10-19T12:03:00.000Z' }); + }); + + it('at: always due when its own schedule runs; inactive triggers never', () => { + const r = resolveTrigger('nightly', { at: 'cron(0 3 * * ? *)', handler }, {}); + expect(dueRun(r, { integrationId: 'a', slotMs: at('2026-10-19T03:00:41Z') })).toEqual({ scheduledFor: '2026-10-19T03:00:00.000Z' }); + const off = resolveTrigger('g', { every: '15m', when: () => false, handler }, {}); + expect(dueRun(off, { integrationId: 'a', slotMs: 0 })).toBeNull(); + }); + + it('slots and hashes are stable, so the same tick delivered twice decides the same', () => { + expect(slotStart(at('2026-10-19T12:04:59Z'), 5)).toBe(at('2026-10-19T12:00:00Z')); + expect(slotStart(at('2026-10-19T12:05:00Z'), 5)).toBe(at('2026-10-19T12:05:00Z')); + expect(hashId('abc')).toBe(hashId('abc')); + expect(hashId('abc')).not.toBe(hashId('abd')); + expect(hashId(42)).toBe(hashId('42')); + }); +}); diff --git a/packages/core/integrations/triggers.js b/packages/core/integrations/triggers.js new file mode 100644 index 000000000..2a61bdd01 --- /dev/null +++ b/packages/core/integrations/triggers.js @@ -0,0 +1,244 @@ +/** + * Recurring triggers (ADR-055): what an integration declares, how one installation resolves it, and whether it is + * due at a tick. Pure; no I/O. The tick itself is `use-cases/run-trigger-tick.js`; the run is + * `IntegrationBase.onTriggerRun`. + * + * static Definition = { + * name: 'housecallpro', + * triggers: { + * pollErp: { every: '15m', handler: 'pollErp' }, // every installation + * ongoingSync: { every: '1h', configurable: { min: '15m' }, handler: sync }, // an installation may retime it + * perPlan: { every: (config) => config.pollEvery, when: (config) => config.poll !== false, handler }, + * digest: { cron: '0 8 * * 1-5', timezone: 'America/New_York', configurable: true, handler }, + * nightly: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler }, // same time for everyone + * }, + * }; + * + * Nothing is created per installation. Devtools provisions one function per integration with one schedule per + * distinct tick and one per `at` trigger (`provisioningOf`). At each tick the function pages through the ENABLED + * installations and sends `{ event: 'FRIGG_TRIGGER_RUN', data: { integrationId, trigger, scheduledFor } }` for each + * one that is due. A disabled or deleted installation is simply not due, and a config change applies at the next tick. + * + * Two additions to ADR-055 (its "Amendment (ADR-063)"): + * - `configurable: true | { min }` on an `every` or `cron` trigger lets an installation choose its own timing at + * `config.triggers.` (`{ every }`, `{ cron, timezone }` or `{ enabled: false }`), never more often than `min`. + * A choice that breaks the floor, or a trigger that is not configurable, keeps the default (and is reported). + * - `cron` (five fields, with `timezone`) is a calendar trigger that may differ per installation. It runs off the + * shared tick: due when a firing falls in (previous tick, this tick]. `at` stays the ADR-055 calendar trigger, + * the same for every installation, with an exact schedule of its own. + */ +const { parseEvery, validateRecurrence, minGapMinutes, lastFiringIn, isTimeZone } = require('../infrastructure/scheduler/recurrence'); + +const TRIGGER_RUN = 'FRIGG_TRIGGER_RUN'; +/** A tick that ran out of time sends this to the integration's queue to carry on from its page cursor. */ +const TRIGGER_TICK = 'FRIGG_TRIGGER_TICK'; + +const DEFAULT_TICK_MINUTES = 5; +const MAX_EVERY_MINUTES = 1440; // longer cadences use a calendar (`at` / `cron`) +const AT = /^cron\(\S+ \S+ \S+ \S+ \S+ \S+\)$/; +const OVERLAP = ['skip', 'allow']; + +const configurableOf = (c) => (c ? (c === true ? {} : c) : false); +const kindOf = (t) => (t.at !== undefined ? 'at' : t.cron !== undefined ? 'cron' : 'every'); +const recurrenceOf = (r) => (r?.every !== undefined ? { every: r.every } : r?.cron !== undefined ? { cron: r.cron, ...(r.timezone ? { timezone: r.timezone } : {}) } : null); + +/** A minutes count as the shortest duration string: 90 -> '90m', 60 -> '1h', 1440 -> '1d'. */ +function durationOf(minutes) { + if (minutes % 1440 === 0) return `${minutes / 1440}d`; + if (minutes % 60 === 0) return `${minutes / 60}h`; + return `${minutes}m`; +} + +/** + * The tick a trigger runs on, in minutes. `tick` when set. Otherwise: for a constant `every`, that interval; for an + * `every` given as a function, 5 minutes; for a `cron`, 5 minutes, or less when the cron itself fires more often. + * A configurable trigger never ticks slower than its floor (or 5 minutes, without one), so the fastest choice it + * allows can run. `at` triggers have no tick (null): each has its own schedule. + */ +function tickMinutesOf(trigger) { + if (kindOf(trigger) === 'at') return null; + if (trigger.tick !== undefined) return parseEvery(trigger.tick); + const configurable = configurableOf(trigger.configurable); + const floor = configurable ? (configurable.min !== undefined ? parseEvery(configurable.min) : DEFAULT_TICK_MINUTES) : Infinity; + let natural; + if (kindOf(trigger) === 'cron') { + const gap = minGapMinutes({ cron: trigger.cron }); + natural = Math.max(1, Math.min(DEFAULT_TICK_MINUTES, gap ?? DEFAULT_TICK_MINUTES)); + } else { + natural = typeof trigger.every === 'function' ? DEFAULT_TICK_MINUTES : parseEvery(trigger.every) ?? DEFAULT_TICK_MINUTES; + } + return Math.max(1, Math.min(natural, floor ?? DEFAULT_TICK_MINUTES, MAX_EVERY_MINUTES)); +} + +/** + * Problems with a trigger declaration, found without an installation (definition time and build time). + * @returns {string[]} + */ +function validateTrigger(name, t) { + const at = (m) => `${name}: ${m}`; + if (!t || typeof t !== 'object') return [at('A trigger is an object')]; + const problems = []; + const kinds = ['every', 'at', 'cron'].filter((k) => t[k] !== undefined); + if (kinds.length !== 1) return [at('Give exactly one of every, at or cron')]; + if (typeof t.handler !== 'function' && typeof t.handler !== 'string') problems.push(at('handler must be a function or a method name')); + if (t.overlap !== undefined && !OVERLAP.includes(t.overlap)) problems.push(at(`overlap must be ${OVERLAP.join(' or ')}`)); + if (t.when !== undefined && typeof t.when !== 'function') problems.push(at('when must be a function of the installation config')); + if (t.tick !== undefined) { + const tick = parseEvery(t.tick); + if (tick === null || tick > MAX_EVERY_MINUTES) problems.push(at(`tick "${t.tick}" is not a duration between 1m and 1d`)); + } + const kind = kinds[0]; + if (kind === 'at') { + if (typeof t.at !== 'string' || !AT.test(t.at)) problems.push(at(`at "${t.at}" must be an EventBridge cron(...) expression with six fields`)); + if (t.timezone !== undefined && !isTimeZone(t.timezone)) problems.push(at(`Unknown time zone "${t.timezone}"`)); + if (t.configurable) problems.push(at('An at trigger runs at one time for every installation; use cron to let installations choose')); + if (t.tick !== undefined) problems.push(at('An at trigger has its own schedule; tick does not apply')); + return problems; + } + const configurable = configurableOf(t.configurable); + if (configurable && configurable.min !== undefined && parseEvery(configurable.min) === null) problems.push(at(`configurable.min "${configurable.min}" is not a duration (15m, 2h, 1d)`)); + if (kind === 'every' && typeof t.every === 'function') return problems; + const recurrence = recurrenceOf(t); + for (const p of validateRecurrence(recurrence)) problems.push(at(p)); + if (kind === 'every' && parseEvery(t.every) > MAX_EVERY_MINUTES) problems.push(at(`every "${t.every}" is longer than 1d; use at or cron`)); + if (!problems.length && configurable && configurable.min !== undefined) { + for (const p of validateRecurrence(recurrence, { min: configurable.min })) problems.push(at(`The default breaks its own minimum: ${p}`)); + } + return problems; +} + +/** Problems with every trigger a Definition declares. */ +function validateTriggers(triggers = {}) { + return Object.entries(triggers || {}).flatMap(([name, t]) => validateTrigger(name, t)); +} + +/** + * What devtools provisions for an integration class: the distinct ticks (minutes) of its `every`/`cron` triggers, + * including the tick of the triggers it resolves per installation (`Definition.instanceTriggers`), and one entry per + * `at` trigger. Throws on an invalid declaration, so a bad trigger fails the build rather than the first tick. + * + * @returns {{ ticks: number[], at: Array<{ name, at, timezone? }> }} + */ +function provisioningOf(IntegrationClass) { + const Definition = IntegrationClass?.Definition || {}; + const triggers = Definition.triggers || {}; + const problems = validateTriggers(triggers); + const instance = Definition.instanceTriggers; + if (instance && parseEvery(instance.tick) === null) problems.push(`instanceTriggers.tick "${instance.tick}" is not a duration`); + if (problems.length) throw new Error(`Integration "${Definition.name}" declares invalid triggers: ${problems.join('; ')}`); + const ticks = new Set(); + const at = []; + for (const [name, t] of Object.entries(triggers)) { + if (kindOf(t) === 'at') at.push({ name, at: t.at, ...(t.timezone ? { timezone: t.timezone } : {}) }); + else ticks.add(tickMinutesOf(t)); + } + if (instance) ticks.add(parseEvery(instance.tick)); + return { ticks: [...ticks].sort((a, b) => a - b), at }; +} + +/** Does this class declare any triggers (fixed or per installation)? */ +function hasTriggers(IntegrationClass) { + const Definition = IntegrationClass?.Definition || {}; + return Object.keys(Definition.triggers || {}).length > 0 || Boolean(Definition.instanceTriggers); +} + +/** + * One trigger, resolved for one installation. + * + * @param {string} name + * @param {Object} t - the declaration (plus `choice` when the caller found the installation's choice elsewhere) + * @param {Object} config - the installation's config + * @returns {{ name, kind, recurrence?, at?, tickMinutes, cadence?, overlap, active: boolean, inactive?: string, + * source: 'definition'|'installation', problems: string[] }} + * `active: false` when `when` excludes the installation or it paused a configurable trigger. + * An invalid choice keeps the default and is listed in `problems`, so a bad edit never stops a trigger. + */ +function resolveTrigger(name, t, config = {}) { + const kind = kindOf(t); + const out = { name, kind, tickMinutes: tickMinutesOf(t), overlap: t.overlap || 'skip', active: true, source: 'definition', problems: [] }; + if (typeof t.when === 'function' && !t.when(config)) return { ...out, active: false, inactive: 'when' }; + if (kind === 'at') return { ...out, at: t.at, timezone: t.timezone }; + + let recurrence = kind === 'every' ? { every: typeof t.every === 'function' ? t.every(config) : t.every } : recurrenceOf(t); + const configurable = configurableOf(t.configurable); + const choice = t.choice !== undefined ? t.choice : config?.triggers?.[name]; + if (choice && typeof choice === 'object' && (recurrenceOf(choice) || choice.enabled !== undefined)) { + if (!configurable) out.problems.push('This trigger is not configurable'); + else { + const picked = recurrenceOf(choice); + const issues = picked ? validateRecurrence(picked, { min: configurable.min }) : []; + if (picked?.every !== undefined && !issues.length && parseEvery(picked.every) > MAX_EVERY_MINUTES) issues.push(`every "${picked.every}" is longer than 1d`); + if (issues.length) out.problems.push(...issues); + else { + if (picked) { recurrence = picked; out.source = 'installation'; } + if (choice.enabled === false) return { ...out, recurrence, active: false, inactive: 'paused' }; + } + } + } + const own = validateRecurrence(recurrence); + if (recurrence.every !== undefined && !own.length && parseEvery(recurrence.every) > MAX_EVERY_MINUTES) own.push(`every "${recurrence.every}" is longer than 1d`); + if (own.length) return { ...out, recurrence, active: false, inactive: 'invalid', problems: [...out.problems, ...own] }; + out.recurrence = recurrence; + // An interval runs every n ticks (rounded up to a whole number of ticks). + if (recurrence.every !== undefined) out.cadence = Math.max(1, Math.ceil(parseEvery(recurrence.every) / out.tickMinutes)); + return out; +} + +/** A stable, evenly spread number for an installation id (FNV-1a, 32 bits). */ +function hashId(value) { + let h = 0x811c9dc5; + for (const ch of String(value)) { + h ^= ch.codePointAt(0); + h = Math.imul(h, 0x01000193) >>> 0; + } + return h; +} + +/** The start of the tick slot an instant falls in: tick runs are keyed by slot, so a late or repeated tick agrees. */ +function slotStart(timeMs, tickMinutes) { + const size = tickMinutes * 60000; + return Math.floor(timeMs / size) * size; +} + +/** + * Is a resolved trigger due for this installation at this tick slot? + * + * - every: due when `slot mod n == hash(integrationId) mod n` (n = cadence in ticks), so it is stateless and spreads + * installations evenly across the cadence. scheduledFor is the slot start. + * - cron: due when it fired in (slot start - tick, slot start]. scheduledFor is that firing, so two ticks in one slot + * (a redelivery, a retried continuation) name the same run. + * - at: due whenever its own schedule invoked the tick; scheduledFor is that time, to the minute. + * + * @param {Object} resolved - from resolveTrigger + * @param {{ integrationId, slotMs: number, firing?: (recurrence, from, to) => number|null }} at + * @returns {{ scheduledFor: string } | null} + */ +function dueRun(resolved, { integrationId, slotMs, firing = lastFiringIn }) { + if (!resolved?.active) return null; + if (resolved.kind === 'at') return { scheduledFor: new Date(Math.floor(slotMs / 60000) * 60000).toISOString() }; + if (resolved.recurrence.every !== undefined) { + const n = resolved.cadence; + const slot = Math.floor(slotMs / (resolved.tickMinutes * 60000)); + return slot % n === hashId(integrationId) % n ? { scheduledFor: new Date(slotMs).toISOString() } : null; + } + const fired = firing(resolved.recurrence, slotMs - resolved.tickMinutes * 60000, slotMs); + return fired === null ? null : { scheduledFor: new Date(fired).toISOString() }; +} + +module.exports = { + TRIGGER_RUN, + TRIGGER_TICK, + DEFAULT_TICK_MINUTES, + configurableOf, + recurrenceOf, + durationOf, + tickMinutesOf, + validateTrigger, + validateTriggers, + provisioningOf, + hasTriggers, + resolveTrigger, + hashId, + slotStart, + dueRun, +}; diff --git a/packages/core/integrations/use-cases/run-trigger-tick.js b/packages/core/integrations/use-cases/run-trigger-tick.js new file mode 100644 index 000000000..bac8547e3 --- /dev/null +++ b/packages/core/integrations/use-cases/run-trigger-tick.js @@ -0,0 +1,132 @@ +/** + * The trigger tick (ADR-055 "Runtime / Tick"): one invocation of an integration's `{integration}Triggers` function. + * + * It is invoked by one of the integration's schedules: `{ tick: '5m', time }` for the shared tick of its `every` and + * `cron` triggers, or `{ at: '', time }` for an `at` trigger. `time` is the schedule's own firing time, + * so the slot it falls in, and therefore every due decision and every `scheduledFor`, is the same if the same tick + * is delivered twice or carried on by a continuation. + * + * 1. Page through the ENABLED installations of this integration, `pageSize` (500) at a time, by id. + * 2. For each installation and trigger, resolve config, `when` and the cadence (integrations/triggers.js). + * 3. Due: `every` when `slot mod n == hash(integrationId) mod n`; `cron` when it fired in (previous tick, this tick]; + * `at` always (it is its own schedule). + * 4. Send `{ event: 'FRIGG_TRIGGER_RUN', data: { integrationId, trigger, scheduledFor } }` to the integration queue in + * batches, with `DelaySeconds` jitter (a stable 0..min(tick, 900 s) per installation and trigger) for tick runs. + * 5. Near the Lambda deadline, send `{ event: 'FRIGG_TRIGGER_TICK', data: { tick|at, time, cursor } }` to the same + * queue and stop; the queue worker carries on from the cursor (IntegrationBase.onTriggerTick). + * + * Missed ticks are not replayed: a trigger is a "check now" signal, and handlers keep their own watermark. + */ +const { parseEvery, lastFiringIn } = require('../../infrastructure/scheduler/recurrence'); +const { TRIGGER_RUN, TRIGGER_TICK, resolveTrigger, dueRun, slotStart, hashId } = require('../triggers'); + +const PAGE_SIZE = 500; +const DEADLINE_MARGIN_MS = 30000; +const MAX_DELAY_SECONDS = 900; // SQS DelaySeconds ceiling + +class RunTriggerTick { + /** + * @param {Object} deps + * @param {Function} deps.integrationClass - the IntegrationBase subclass (Definition.name, triggersFor) + * @param {Object} deps.integrationRepository - findIntegrations({ type, status, limit, after }) + * @param {Object} deps.queue - { batchSend(entries, queueUrl, { delaySeconds }), send(message, queueUrl) } + * @param {string} deps.queueUrl + * @param {Object} [deps.logger] + * @param {number} [deps.pageSize] + */ + constructor({ integrationClass, integrationRepository, queue, queueUrl, logger, pageSize = PAGE_SIZE }) { + this.Integration = integrationClass; + this.repository = integrationRepository; + this.queue = queue; + this.queueUrl = queueUrl; + this.logger = logger; + this.pageSize = pageSize; + } + + /** + * @param {Object} params + * @param {string} [params.tick] - '5m': evaluate the every/cron triggers on this tick + * @param {string} [params.at] - an `at` trigger's name: run it for every ENABLED installation + * @param {string} [params.time] - the schedule's firing time (ISO); defaults to now + * @param {string|number} [params.cursor] - continue after this installation id + * @param {() => number} [params.remainingMs] - time left in this invocation + * @returns {Promise<{ installations: number, sent: number, pages: number, continued: boolean, failed: Array }>} + */ + async execute({ tick, at, time, cursor, remainingMs = () => Infinity }) { + const name = this.Integration.Definition?.name; + if (!this.queueUrl) throw new Error(`No queue URL for integration "${name}"; trigger runs have nowhere to go`); + const tickMinutes = at ? null : parseEvery(tick); + if (!at && tickMinutes === null) throw new Error(`Trigger tick "${tick}" is not a duration`); + const timeMs = time && !Number.isNaN(Date.parse(time)) ? Date.parse(time) : Date.now(); + const slotMs = at ? Math.floor(timeMs / 60000) * 60000 : slotStart(timeMs, tickMinutes); + const firings = new Map(); // one cron answer per recurrence per tick (the window is the same), not per installation + const firing = (recurrence, from, to) => { + const key = `${recurrence.cron}|${recurrence.timezone || 'UTC'}`; + if (!firings.has(key)) firings.set(key, lastFiringIn(recurrence, from, to)); + return firings.get(key); + }; + const delayOf = (m) => (at ? 0 : hashId(`${m.data.integrationId}:${m.data.trigger}`) % Math.min(tickMinutes * 60, MAX_DELAY_SECONDS)); + + const out = { installations: 0, sent: 0, pages: 0, continued: false, failed: [] }; + let after = cursor; + for (;;) { + const rows = await this.repository.findIntegrations({ type: name, status: 'ENABLED', limit: this.pageSize, after }); + out.pages += 1; + const messages = []; + for (const row of rows) { + out.installations += 1; + try { + const declarations = await this.Integration.triggersFor({ config: row.config || {}, record: row }); + for (const declaration of declarations) { + const isAt = declaration.at !== undefined; + if (at ? declaration.name !== at || !isAt : isAt) continue; + const resolved = resolveTrigger(declaration.name, declaration, row.config || {}); + if (!at && resolved.tickMinutes !== tickMinutes) continue; + const due = dueRun(resolved, { integrationId: row.id, slotMs, firing }); + if (due) messages.push({ event: TRIGGER_RUN, data: { integrationId: row.id, trigger: declaration.name, scheduledFor: due.scheduledFor } }); + } + } catch (error) { + // One installation's bad config must not stop the others' runs. + out.failed.push({ integrationId: row.id, error: error.message }); + this.logger?.warn('Trigger tick skipped an installation', { eventName: 'frigg.triggers.installation_failed', integration: name, integrationId: row.id, error }); + } + } + if (messages.length) { + await this.queue.batchSend(messages, this.queueUrl, { delaySeconds: delayOf }); + out.sent += messages.length; + } + if (rows.length < this.pageSize) return out; + after = rows[rows.length - 1].id; + if (remainingMs() < DEADLINE_MARGIN_MS) { + await this.queue.send({ event: TRIGGER_TICK, data: { ...(at ? { at } : { tick }), time: new Date(timeMs).toISOString(), cursor: after } }, this.queueUrl); + out.continued = true; + return out; + } + } + } +} + +/** The queue URL Frigg's infrastructure gives an integration: `_QUEUE_URL`, dashes as underscores. */ +function queueUrlFor(integrationName) { + return process.env[`${String(integrationName).toUpperCase().replace(/-/g, '_')}_QUEUE_URL`]; +} + +/** + * Run a tick with Frigg's real adapters: the integration repository, the SQS queuer and the integration's queue. + * Used by the `{integration}Triggers` Lambda and by a continuation on the queue worker. + */ +async function runTriggerTick(integrationClass, event = {}, context) { + const { createIntegrationRepository } = require('../repositories/integration-repository-factory'); + const { QueuerUtil } = require('../../queues'); + const { getLogger } = require('../../logs'); + const remainingMs = typeof context?.getRemainingTimeInMillis === 'function' ? () => context.getRemainingTimeInMillis() : undefined; + return new RunTriggerTick({ + integrationClass, + integrationRepository: createIntegrationRepository(), + queue: QueuerUtil, + queueUrl: queueUrlFor(integrationClass.Definition.name), + logger: getLogger('frigg.triggers'), + }).execute({ tick: event.tick, at: event.at, time: event.time, cursor: event.cursor, remainingMs }); +} + +module.exports = { RunTriggerTick, runTriggerTick, queueUrlFor, PAGE_SIZE }; diff --git a/packages/core/integrations/use-cases/run-trigger.js b/packages/core/integrations/use-cases/run-trigger.js new file mode 100644 index 000000000..a03d53687 --- /dev/null +++ b/packages/core/integrations/use-cases/run-trigger.js @@ -0,0 +1,77 @@ +/** + * Run one recurring trigger for one installation (ADR-055 "Run"). The queue worker has already hydrated the + * installation by `integrationId`; this decides whether the run happens and records it. + * + * 1. The installation must be ENABLED; anything else discards the run. + * 2. With `overlap: 'skip'` (the default) a Process record per (installation, trigger), type `TRIGGER:`, is + * the lease: a run whose record is not terminal and was touched within 2 x tick is still going, so this run is + * skipped and acked. Otherwise the record is reused (one row per installation and trigger, not one per run), set + * RUNNING, and set COMPLETED or ERROR when the handler ends. A failed run is marked ERROR before the error is + * rethrown, so the worker's retry is not skipped by the run it is retrying. `overlap: 'allow'` keeps no record. + * 3. Errors go back to the queue worker: its retry, halt and DLQ rules apply as for any queued event. + */ +const TERMINAL = ['COMPLETED', 'ERROR']; +const AT_LEASE_MINUTES = 60; // an `at` trigger has no tick; its lease is two hours + +class RunTrigger { + /** + * @param {Object} deps + * @param {Object} deps.processRepository - findByIntegrationAndType, create, update + * @param {() => number} [deps.now] + */ + constructor({ processRepository, now = () => Date.now() }) { + this.processRepository = processRepository; + this.now = now; + } + + /** + * @param {Object} params + * @param {Object} params.integration - hydrated instance: id, userId, status, telemetry + * @param {Object} params.resolved - resolveTrigger(...) for this installation + * @param {(payload) => Promise} params.invoke - calls the trigger's handler + * @param {{ trigger: string, scheduledFor: string }} params.payload + * @returns {Promise<{ status: 'ran'|'skipped'|'discarded', reason?: string, result?: any }>} + */ + async execute({ integration, resolved, invoke, payload }) { + const count = (name) => { try { integration.telemetry?.count(name, 1, { trigger: payload.trigger }); } catch (_) { /* never fails a run */ } }; + if (integration.status !== 'ENABLED') return { status: 'discarded', reason: `installation is ${integration.status}` }; + if (!resolved?.active) return { status: 'discarded', reason: resolved ? `trigger is ${resolved.inactive}` : 'trigger is not declared' }; + + if (resolved.overlap === 'allow') return this.run({ invoke, payload, count }); + + const type = `TRIGGER:${payload.trigger}`; + const [existing] = await this.processRepository.findByIntegrationAndType(integration.id, type); + const leaseMs = 2 * (resolved.tickMinutes || AT_LEASE_MINUTES) * 60000; + const touched = existing ? new Date(existing.updatedAt || existing.createdAt || 0).getTime() : 0; + if (existing && !TERMINAL.includes(existing.state) && this.now() - touched < leaseMs) { + count('trigger.skipped'); + return { status: 'skipped', reason: `a run of ${payload.trigger} is still going` }; + } + + const context = { scheduledFor: payload.scheduledFor, startedAt: new Date(this.now()).toISOString() }; + const record = existing + ? await this.processRepository.update(existing.id, { state: 'RUNNING', context }) + : await this.processRepository.create({ userId: integration.userId, integrationId: integration.id, name: type, type, state: 'RUNNING', context, results: {} }); + try { + const out = await this.run({ invoke, payload, count }); + await this.processRepository.update(record.id, { state: 'COMPLETED', results: { scheduledFor: payload.scheduledFor, finishedAt: new Date(this.now()).toISOString() } }); + return out; + } catch (error) { + await this.processRepository.update(record.id, { state: 'ERROR', results: { scheduledFor: payload.scheduledFor, error: error.message } }).catch(() => {}); + throw error; + } + } + + async run({ invoke, payload, count }) { + try { + const result = await invoke(payload); + count('trigger.run'); + return { status: 'ran', result }; + } catch (error) { + count('trigger.failed'); + throw error; + } + } +} + +module.exports = { RunTrigger, TERMINAL }; diff --git a/packages/core/queues/queuer-util.js b/packages/core/queues/queuer-util.js index 0c9f502ca..c6a1fd157 100644 --- a/packages/core/queues/queuer-util.js +++ b/packages/core/queues/queuer-util.js @@ -88,14 +88,21 @@ const QueuerUtil = { return result; }, - batchSend: async (entries = [], queueUrl) => { + /** + * Send entries in batches of 10. `options.delaySeconds(entry, index)` may return a per-message SQS + * DelaySeconds (0-900), e.g. to spread trigger runs across a tick. + */ + batchSend: async (entries = [], queueUrl, options = {}) => { const buffer = []; const batchSize = 10; + const delayOf = typeof options.delaySeconds === 'function' ? options.delaySeconds : null; - for (const entry of entries) { + for (const [index, entry] of entries.entries()) { + const delay = delayOf ? Math.max(0, Math.min(900, Math.floor(delayOf(entry, index) || 0))) : 0; buffer.push({ Id: uuid(), MessageBody: JSON.stringify(entry), + ...(delay ? { DelaySeconds: delay } : {}), }); // Sends 10, then purges the buffer if (buffer.length === batchSize) { diff --git a/packages/core/queues/queuer-util.test.js b/packages/core/queues/queuer-util.test.js index e0d6fce6d..0c171113b 100644 --- a/packages/core/queues/queuer-util.test.js +++ b/packages/core/queues/queuer-util.test.js @@ -70,6 +70,19 @@ describe('QueuerUtil - AWS SDK v3', () => { expect(call.args[0].input.QueueUrl).toBe(queueUrl); }); + it('sets a per-message DelaySeconds when asked, clamped to 0-900, and none otherwise', async () => { + sqsMock.on(SendMessageBatchCommand).resolves({ Successful: [], Failed: [] }); + const entries = [{ n: 0 }, { n: 1 }, { n: 2 }]; + await QueuerUtil.batchSend(entries, 'https://q', { delaySeconds: (e) => [0, 42, 5000][e.n] }); + const sent = sqsMock.call(0).args[0].input.Entries; + expect(sent.map((e) => e.DelaySeconds)).toEqual([undefined, 42, 900]); + expect(JSON.parse(sent[1].MessageBody)).toEqual({ n: 1 }); + + sqsMock.resetHistory(); + await QueuerUtil.batchSend(entries, 'https://q'); + expect(sqsMock.call(0).args[0].input.Entries.every((e) => e.DelaySeconds === undefined)).toBe(true); + }); + it('should send multiple batches for large entry sets (10 per batch)', async () => { sqsMock.on(SendMessageBatchCommand).resolves({ Successful: [], diff --git a/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js b/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js index cd31e4aa8..daaf5c03b 100644 --- a/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js +++ b/packages/devtools/infrastructure/domains/scheduler/scheduler-builder.js @@ -8,9 +8,8 @@ * - Creating IAM Role for EventBridge Scheduler to send messages to SQS * - Adding necessary IAM statements for Lambda to create/delete schedules * - * This builder enables integrations to schedule one-time jobs (e.g., webhook renewals) and - * recurring per-installation jobs (Definition.schedules, data-defined schedule triggers) - * using AWS EventBridge Scheduler. Recurring jobs are upserted, hence UpdateSchedule. + * This builder enables integrations to schedule one-time jobs (e.g., webhook renewals) + * using AWS EventBridge Scheduler. */ const { InfrastructureBuilder, ValidationResult } = require('../shared/base-builder'); @@ -174,7 +173,6 @@ class SchedulerBuilder extends InfrastructureBuilder { Effect: 'Allow', Action: [ 'scheduler:CreateSchedule', - 'scheduler:UpdateSchedule', 'scheduler:DeleteSchedule', 'scheduler:GetSchedule', ], diff --git a/packages/devtools/infrastructure/domains/scheduler/trigger-builder.js b/packages/devtools/infrastructure/domains/scheduler/trigger-builder.js new file mode 100644 index 000000000..1eeeb968c --- /dev/null +++ b/packages/devtools/infrastructure/domains/scheduler/trigger-builder.js @@ -0,0 +1,114 @@ +/** + * Trigger Builder (ADR-055 §2) + * + * Domain Layer - Hexagonal Architecture + * + * One function per integration that declares recurring triggers: `{integration}Triggers`, with the Prisma layer. + * It gets one serverless `schedule` event per distinct tick of the integration's `every` / `cron` triggers (and of + * the triggers it resolves per installation, `Definition.instanceTriggers`), plus one EventBridge Scheduler event per + * `at` trigger, with its time zone. Each tick pages through the ENABLED installations and queues the runs that are + * due (core: integrations/use-cases/run-trigger-tick.js). Nothing is created per installation, and these resources + * change only when a Definition does, so they live in CloudFormation and leave with the stack. + * + * IAM, and why this builder adds none: + * - A `rate()` tick is an EventBridge rule; the framework adds the Lambda permission for it. + * - An `at` trigger uses `method: scheduler`; the framework adds `scheduler.amazonaws.com` to the function role's + * trust policy and `lambda:InvokeFunction` on this function to it. + * - The tick reads the integration table (the Prisma layer and the provider's database access, like the queue + * worker) and sends to its own `Queue`, which the provider-level `sqs:SendMessage(Batch)` statement on + * `${service}--${stage}-*Queue` already allows. It never calls EventBridge Scheduler at runtime, so it needs none + * of SchedulerBuilder's scheduler:* permissions (those stay for one-time jobs). + */ + +const { InfrastructureBuilder, ValidationResult } = require('../shared/base-builder'); +const { IntegrationBuilder } = require('../integration/integration-builder'); +const { + hasTriggers, + provisioningOf, + durationOf, +} = require('@friggframework/core/integrations/triggers'); + +/** EventBridge rate() for a tick in minutes: rate(1 minute), rate(5 minutes), rate(1 hour), rate(2 hours), rate(1 day). */ +function rateExpression(minutes) { + const [count, unit] = + minutes % 1440 === 0 ? [minutes / 1440, 'day'] : minutes % 60 === 0 ? [minutes / 60, 'hour'] : [minutes, 'minute']; + return `rate(${count} ${unit}${count === 1 ? '' : 's'})`; +} + +/** The function name the trigger tick for an integration runs as. Wire contract with function-environments.js. */ +function triggerFunctionName(integrationName) { + return `${integrationName}Triggers`; +} + +class TriggerBuilder extends InfrastructureBuilder { + constructor() { + super(); + this.name = 'TriggerBuilder'; + } + + shouldExecute(appDefinition) { + return Array.isArray(appDefinition.integrations) && appDefinition.integrations.some((i) => hasTriggers(i)); + } + + getDependencies() { + return ['IntegrationBuilder']; // the tick sends to the integration queue + } + + validate(appDefinition) { + const result = new ValidationResult(); + for (const integration of appDefinition.integrations || []) { + if (!hasTriggers(integration)) continue; + try { + provisioningOf(integration); + } catch (error) { + result.addError(error.message); + } + } + return result; + } + + async build(appDefinition) { + console.log(`\n[${this.name}] Configuring recurring triggers...`); + const result = { functions: {}, resources: {}, environment: {}, custom: {}, iamStatements: [] }; + const usePrismaLayer = appDefinition.usePrismaLambdaLayer !== false; + const functionPackageConfig = new IntegrationBuilder().createFunctionPackageConfig(usePrismaLayer, appDefinition); + + for (const integration of appDefinition.integrations || []) { + if (!hasTriggers(integration)) continue; + const name = integration.Definition.name; + const { ticks, at } = provisioningOf(integration); + const events = [ + ...ticks.map((minutes) => ({ + schedule: { + rate: [rateExpression(minutes)], + // The rule's own firing time goes in, so a repeated delivery falls in the same tick slot. + inputTransformer: { + inputPathsMap: { time: '$.time' }, + inputTemplate: `{"tick":"${durationOf(minutes)}","time":