From e089bb7ad526e4a730312eb2d40d3534b2b89054 Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:45:35 +0200 Subject: [PATCH 1/4] fix: the post's about panel no longer nests links, and a bounce failure propagates --- site/app/routes/post.tsx | 30 ++++++++++++++---------------- site/app/styles/pages/blog.css | 3 +++ site/server/sends.test.ts | 4 +++- site/server/sends.ts | 4 ++-- site/server/subscribers.ts | 4 ++-- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/site/app/routes/post.tsx b/site/app/routes/post.tsx index fae2a88..b602b38 100644 --- a/site/app/routes/post.tsx +++ b/site/app/routes/post.tsx @@ -94,23 +94,21 @@ export default function Post({ loaderData }: Route.ComponentProps) { )} {project && ( - +
- - - {project.name} - {project.summary} - - + + + + {project.name} + {project.summary} + + +
)} {more.length > 0 && ( diff --git a/site/app/styles/pages/blog.css b/site/app/styles/pages/blog.css index 251ab87..b94d4d9 100644 --- a/site/app/styles/pages/blog.css +++ b/site/app/styles/pages/blog.css @@ -638,6 +638,9 @@ .window.mini:hover { border-color: var(--color-muted); } +.window.mini .face { + display: grid; +} .window.mini .cover { aspect-ratio: 16 / 6; } diff --git a/site/server/sends.test.ts b/site/server/sends.test.ts index 004255a..6b6697a 100644 --- a/site/server/sends.test.ts +++ b/site/server/sends.test.ts @@ -33,7 +33,9 @@ function setup(deliver: (message: Outgoing) => Promise, perHour = 20) { compose: async (send) => send.ref === 'skip' ? null : message(`${send.subscriber}@example.com`), perHour: () => perHour, - bounced: (subscriber) => bounced.push(subscriber), + bounced: async (subscriber) => { + bounced.push(subscriber); + }, }); return { db, jobs, sends, bounced }; } diff --git a/site/server/sends.ts b/site/server/sends.ts index daa50fa..fcdc9c0 100644 --- a/site/server/sends.ts +++ b/site/server/sends.ts @@ -68,7 +68,7 @@ export function createSends( deliver: Deliver; compose: (send: SendRow) => Promise; perHour: () => number; - bounced: (subscriber: string, reason: string, at: string) => void; + bounced: (subscriber: string, reason: string, at: string) => Promise; }, ) { const columns = 'id, subscriber, kind, ref, due_at, sent_at, status, attempts, error FROM sends'; @@ -124,7 +124,7 @@ export function createSends( const failure = classify(error); if (failure.bounce) { finish(send.id, 'bounced', failure.message, at); - deps.bounced(send.subscriber, failure.message, at); + await deps.bounced(send.subscriber, failure.message, at); return 'bounced'; } if (failure.permanent || attempts >= MAX_ATTEMPTS) { diff --git a/site/server/subscribers.ts b/site/server/subscribers.ts index ab4d4bc..d9b54f0 100644 --- a/site/server/subscribers.ts +++ b/site/server/subscribers.ts @@ -231,13 +231,13 @@ export function createSubscribers( deliver: deps.deliver, compose, perHour: () => deps.settings()['email.hourly'], - bounced(id, reason, at) { + async bounced(id, reason, at) { db.prepare( 'UPDATE subscribers SET suppressed_at = ?, suppressed_reason = ?, updated_at = ? WHERE id = ?', ).run(at, reason, at, id); const subscriber = read(id); if (subscriber) { - changed('bounced', subscriber, 'An address bounced and was suppressed', at).catch(() => {}); + await changed('bounced', subscriber, 'An address bounced and was suppressed', at); } }, }); From 1182daafbe6ca83077fba052823c51418ac5a95c Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:10:12 +0200 Subject: [PATCH 2/4] feat(live): one topic map with its audit, presence, reading and editing badges, view counters and live sign-out --- site/app/components/kit/page.tsx | 2 - site/app/context.ts | 4 + site/app/live.ts | 185 +++++++++++++++++++++++--- site/app/pages/member-page.tsx | 2 - site/app/pages/project-page.tsx | 17 ++- site/app/root.tsx | 16 ++- site/app/routes/admin.tsx | 28 ++-- site/app/routes/author.tsx | 3 +- site/app/routes/blog-mine.tsx | 3 +- site/app/routes/blog-new.tsx | 3 +- site/app/routes/blog.tsx | 3 +- site/app/routes/changelog.tsx | 3 + site/app/routes/editor.tsx | 3 +- site/app/routes/epics.tsx | 5 +- site/app/routes/home.tsx | 5 +- site/app/routes/kit.tsx | 3 + site/app/routes/member-changelog.tsx | 3 + site/app/routes/member.tsx | 5 +- site/app/routes/members.tsx | 12 +- site/app/routes/messages.tsx | 3 +- site/app/routes/not-found.tsx | 3 + site/app/routes/post-editor.tsx | 3 +- site/app/routes/post.tsx | 23 +++- site/app/routes/privacy.tsx | 20 ++- site/app/routes/project-changelog.tsx | 3 + site/app/routes/projects.tsx | 5 +- site/app/routes/resolved.tsx | 12 +- site/app/routes/search.tsx | 3 + site/app/routes/sign-in.tsx | 3 + site/app/routes/status.tsx | 10 +- site/app/routes/subscribe.tsx | 3 +- site/app/routes/subscription.tsx | 3 +- site/app/topics.ts | 112 ++++++++++++++++ site/app/types.ts | 10 ++ site/server/address.ts | 20 ++- site/server/admin.ts | 12 +- site/server/app.ts | 18 +++ site/server/auth.test.ts | 15 ++- site/server/collab.ts | 73 ++++++++-- site/server/live-audit.test.ts | 144 ++++++++++++++++++++ site/server/live.test.ts | 47 ++++++- site/server/live.ts | 117 +++++++++++++--- site/server/main.ts | 25 +++- site/server/migrations/008-views.sql | 7 + site/server/presence.test.ts | 79 +++++++++++ site/server/presence.ts | 177 ++++++++++++++++++++++++ site/server/sync.ts | 11 +- site/server/testing.ts | 54 ++++++++ site/server/views.test.ts | 74 +++++++++++ site/server/views.ts | 100 ++++++++++++++ 50 files changed, 1379 insertions(+), 115 deletions(-) create mode 100644 site/app/topics.ts create mode 100644 site/server/live-audit.test.ts create mode 100644 site/server/migrations/008-views.sql create mode 100644 site/server/presence.test.ts create mode 100644 site/server/presence.ts create mode 100644 site/server/views.test.ts create mode 100644 site/server/views.ts diff --git a/site/app/components/kit/page.tsx b/site/app/components/kit/page.tsx index cb5741d..80e8361 100644 --- a/site/app/components/kit/page.tsx +++ b/site/app/components/kit/page.tsx @@ -87,5 +87,3 @@ export function StandardRail({ running, lines }: { running: Running[]; lines: Co ); } - -export const RAIL_TOPICS = ['deploys', 'releases', 'projects', 'uptime']; diff --git a/site/app/context.ts b/site/app/context.ts index 7b60476..997f4cc 100644 --- a/site/app/context.ts +++ b/site/app/context.ts @@ -5,6 +5,7 @@ import type { Collab } from '../server/collab.ts'; import type { Capture } from '../server/errors.ts'; import type { Messages } from '../server/messages.ts'; import type { Posts } from '../server/posts.ts'; +import type { Presence } from '../server/presence.ts'; import type { Presentations } from '../server/presentation.ts'; import type { Profiles } from '../server/profiles.ts'; import type { Writeback } from '../server/writeback.ts'; @@ -12,6 +13,7 @@ import type { Uploads } from '../server/uploads.ts'; import type { Share } from '../server/share.ts'; import type { Reads } from '../server/reads.ts'; import type { Subscribers } from '../server/subscribers.ts'; +import type { Views } from '../server/views.ts'; import type { Values } from './settings'; import type { Directory, Viewer } from './types'; @@ -29,6 +31,8 @@ export interface Hub { posts: Posts; subscribers: Subscribers; collab: Collab; + presence: Presence; + views: Views; address: string; origin: string; viewer: Viewer | null; diff --git a/site/app/live.ts b/site/app/live.ts index 4c83332..d9020f9 100644 --- a/site/app/live.ts +++ b/site/app/live.ts @@ -1,6 +1,7 @@ -import { useEffect, useEffectEvent } from 'react'; +import { useEffect, useEffectEvent, useSyncExternalStore } from 'react'; import { useMatches, useRevalidator } from 'react-router'; -import type { ConsoleLine } from './types'; +import { revalidates, subscription, type LiveHandle } from './topics'; +import type { ConsoleLine, Presence } from './types'; const COALESCE_MS = 100; @@ -10,6 +11,141 @@ export interface LiveMessage { key?: string; kind: string; console: Omit | null; + data?: Record; +} + +export interface Editor { + handle: string; + name: string; + color: string; +} + +function createStore(initial: T) { + let value = initial; + const subscribers = new Set<() => void>(); + return { + get: () => value, + initial: () => initial, + set(next: T) { + value = next; + subscribers.forEach((notify) => notify()); + }, + subscribe(notify: () => void) { + subscribers.add(notify); + return () => { + subscribers.delete(notify); + }; + }, + }; +} + +const connection = createStore<'live' | 'reconnecting'>('live'); +const presence = createStore(null); +const reading = createStore(new Map()); +const editing = createStore(new Map()); +const views = createStore(new Map()); + +function useStore(store: ReturnType>) { + return useSyncExternalStore(store.subscribe, store.get, store.initial); +} + +export function useConnection() { + return useStore(connection); +} + +export function usePresence(initial: Presence) { + return useStore(presence) ?? initial; +} + +export function useOnline() { + return new Set(useStore(presence)?.online ?? []); +} + +export function useReading(key: string, initial: number) { + return useStore(reading).get(key) ?? initial; +} + +export function useEditing(key: string, initial: Editor[]) { + return useStore(editing).get(key) ?? initial; +} + +export function useViews(totals: Record) { + const live = useStore(views); + return (kind: 'post' | 'project', key: string) => + live.get(`${kind}:${key}`) ?? totals[`${kind}:${key}`] ?? 0; +} + +function patch(message: LiveMessage) { + const data = message.data ?? {}; + switch (message.topic) { + case 'presence': + presence.set(data as unknown as Presence); + break; + case 'reading': + if (message.key) { + reading.set(new Map(reading.get()).set(message.key, Number(data.reading))); + } + break; + case 'editing': + if (message.key) { + editing.set(new Map(editing.get()).set(message.key, data.editors as Editor[])); + } + break; + case 'views': + views.set(new Map(views.get()).set(`${data.kind}:${data.key}`, Number(data.total))); + break; + } +} + +const VIEW_AFTER_MS = 10_000; +const VIEW_SCROLL = 0.25; + +export function useView(kind: 'post' | 'project', key: string) { + useEffect(() => { + let sent = false; + let scrolled = false; + let visibleFor = 0; + let since = document.visibilityState === 'visible' ? performance.now() : null; + let timer: ReturnType | undefined; + const elapsed = () => visibleFor + (since === null ? 0 : performance.now() - since); + const check = () => { + clearTimeout(timer); + timer = undefined; + if (sent) { + return; + } + if (elapsed() >= VIEW_AFTER_MS && scrolled) { + sent = true; + void fetch(`/views/${kind}/${key}`, { method: 'POST', keepalive: true }); + } else if (since !== null) { + timer = setTimeout(check, Math.max(50, VIEW_AFTER_MS - elapsed())); + } + }; + const onScroll = () => { + if (!scrolled) { + scrolled = (scrollY + innerHeight) / document.documentElement.scrollHeight >= VIEW_SCROLL; + check(); + } + }; + const onVisibility = () => { + if (document.visibilityState === 'visible') { + since = performance.now(); + } else if (since !== null) { + visibleFor += performance.now() - since; + since = null; + } + check(); + }; + addEventListener('scroll', onScroll, { passive: true }); + document.addEventListener('visibilitychange', onVisibility); + onScroll(); + check(); + return () => { + clearTimeout(timer); + removeEventListener('scroll', onScroll); + document.removeEventListener('visibilitychange', onVisibility); + }; + }, [kind, key]); } const listeners = new Set<(message: LiveMessage) => void>(); @@ -26,15 +162,12 @@ export function useLiveMessages(listener: (message: LiveMessage) => void) { } export function useLive() { - const topics = [ - ...new Set( - useMatches().flatMap( - (match) => (match.handle as { topics?: string[] } | undefined)?.topics ?? [], - ), - ), - ] - .sort() - .join(','); + const matches = useMatches(); + const live = subscription( + matches.map((match) => (match.handle as LiveHandle | undefined) ?? { topics: [] }), + matches, + ); + const topics = live.topics.join(','); const { revalidate } = useRevalidator(); const refresh = useEffectEvent(() => revalidate()); @@ -42,19 +175,41 @@ export function useLive() { if (!topics) { return; } - const source = new EventSource(`/live?topics=${encodeURIComponent(topics)}`); + reading.set(new Map()); + editing.set(new Map()); + const query = new URLSearchParams({ topics }); + if (live.reading) { + query.set('reading', live.reading); + } + const source = new EventSource(`/live?${query}`); let timer: ReturnType | undefined; + let dropped = false; + source.onopen = () => { + connection.set('live'); + if (dropped) { + dropped = false; + refresh(); + } + }; + source.onerror = () => { + dropped = true; + connection.set('reconnecting'); + }; source.onmessage = (event: MessageEvent) => { const message = { ...JSON.parse(event.data), id: event.lastEventId } as LiveMessage; + patch(message); for (const listener of listeners) { listener(message); } - clearTimeout(timer); - timer = setTimeout(refresh, COALESCE_MS); + if (revalidates(message.topic)) { + clearTimeout(timer); + timer = setTimeout(refresh, COALESCE_MS); + } }; return () => { clearTimeout(timer); source.close(); + connection.set('live'); }; - }, [topics]); + }, [topics, live.reading]); } diff --git a/site/app/pages/member-page.tsx b/site/app/pages/member-page.tsx index a4973b3..494298d 100644 --- a/site/app/pages/member-page.tsx +++ b/site/app/pages/member-page.tsx @@ -11,8 +11,6 @@ import { useHub, useRoot } from '../hub'; import { createModel } from '../model'; import type { Member } from '../types'; -export const MEMBER_TOPICS = ['deploys', 'releases', 'projects', 'uptime', 'profiles', 'posts']; - export function loadMember({ reads, directory, profiles, viewer }: Hub, member: Member) { const owned = new Set( createModel(directory) diff --git a/site/app/pages/project-page.tsx b/site/app/pages/project-page.tsx index 6a666e0..481038f 100644 --- a/site/app/pages/project-page.tsx +++ b/site/app/pages/project-page.tsx @@ -1,6 +1,7 @@ import { Link } from 'react-router'; import AuthorLink from '../components/AuthorLink'; import { accentStyle, Avatar, Button, Cover, Panel } from '../components/kit/basics'; +import { ReadingBadge } from '../components/kit/live'; import { Fact, PageHeader } from '../components/kit/page'; import { ChangelogPanel, RunningPanel } from '../components/kit/panels'; import { PostCard, WritingStrip } from '../components/kit/posts'; @@ -15,15 +16,15 @@ import TagList from '../components/TagList'; import type { Hub } from '../context'; import { day, shortTag } from '../format'; import { useHub, useRoot } from '../hub'; +import { useView, useViews } from '../live'; import { changelogPath } from '../model'; import type { Deploy, Project } from '../types'; -export const PROJECT_TOPICS = ['deploys', 'releases', 'projects', 'uptime', 'posts']; - -export function loadProject({ reads, directory, viewer }: Hub, project: Project) { +export function loadProject({ reads, directory, viewer, presence }: Hub, project: Project) { const owner = directory.members.find((member) => member.handle === project.owner); return { page: 'project' as const, + reading: presence.reading(project.slug), editable: Boolean( viewer && (viewer.owner || viewer.login.toLowerCase() === owner?.github.toLowerCase()), ), @@ -61,6 +62,8 @@ export function ProjectPage({ data: loaderData }: { data: ReturnType hub.postById(id) ?? []); const writing = hub.postsAbout(project); + const views = useViews(useRoot()!.views)('project', project.slug); + useView('project', project.slug); return ( } title={project.name} - aside={} + aside={ + <> + + + + } actions={ <> {loaderData.editable && owner && ( @@ -142,6 +150,7 @@ export function ProjectPage({ data: loaderData }: { data: ReturnType} {releases[0] && } + {views > 0 && } } > diff --git a/site/app/root.tsx b/site/app/root.tsx index e305e40..65e8006 100644 --- a/site/app/root.tsx +++ b/site/app/root.tsx @@ -15,17 +15,23 @@ import { useLive } from './live'; import { metaFor, metaTags } from './meta'; import { createModel } from './model'; import { shareCard, sharePath } from './share'; +import { ROOT_TOPICS, type LiveHandle } from './topics'; import './styles/global.css'; import.meta.glob('./styles/pages/*.css', { eager: true }); -export const handle = { - topics: ['members', 'profiles', 'projects', 'releases', 'settings', 'posts'], -}; +export const handle: LiveHandle = { topics: ROOT_TOPICS }; export function loader({ context }: Route.LoaderArgs) { - const { directory, settings, viewer, csrf } = context.get(hubContext); - return { directory, settings, viewer, csrf }; + const { directory, settings, viewer, csrf, presence, views } = context.get(hubContext); + return { + directory, + settings, + viewer, + csrf, + presence: presence.current(), + views: views.totals(), + }; } export function meta({ loaderData, location }: Route.MetaArgs) { diff --git a/site/app/routes/admin.tsx b/site/app/routes/admin.tsx index 88fe472..4478d82 100644 --- a/site/app/routes/admin.tsx +++ b/site/app/routes/admin.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState, type CSSProperties } from 'react'; -import { data, Link, useFetcher } from 'react-router'; +import { data, Link, redirect, useFetcher } from 'react-router'; import type { Route } from './+types/admin'; import { Avatar, Band, Button, Panel } from '../components/kit/basics'; import { Confirm, Segmented, Sortable, Switch } from '../components/kit/controls'; @@ -22,8 +22,9 @@ import { type TextSetting, } from '../settings'; import { describeScope, parseScope, validEmail } from '../subscribe'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['settings', 'errors', 'subscribers'] }; +export const handle = topicsFor('admin'); function owner(hub: Hub) { if (!hub.viewer?.owner) { @@ -69,7 +70,8 @@ export async function action({ context, request }: Route.ActionArgs) { await hub.admin.resolveError(String(form.get('fingerprint')), viewer.login); break; case 'sign-out-everyone': - return { ok: `${await hub.admin.signOutEveryone(viewer.login)} sessions ended.` }; + await hub.admin.signOutEveryone(viewer.login); + throw redirect('/'); case 'test-email': { const to = String(form.get('to') ?? '') .trim() @@ -83,6 +85,9 @@ export async function action({ context, request }: Route.ActionArgs) { } } } catch (error) { + if (error instanceof Response) { + throw error; + } return data( { error: error instanceof Error ? error.message : String(error), key }, { status: 400 }, @@ -575,17 +580,12 @@ function Sessions({ csrf }: { csrf: string }) { const signOut = useFetcher(); return ( -
- signOut.submit({ csrf, intent: 'sign-out-everyone' }, { method: 'post' })} - /> - {signOut.data && 'ok' in signOut.data && ( - {signOut.data.ok} - )} -
+ signOut.submit({ csrf, intent: 'sign-out-everyone' }, { method: 'post' })} + /> ); } diff --git a/site/app/routes/author.tsx b/site/app/routes/author.tsx index aee45c4..0378305 100644 --- a/site/app/routes/author.tsx +++ b/site/app/routes/author.tsx @@ -8,8 +8,9 @@ import { Page } from '../components/Shell'; import { hubContext } from '../context'; import { useHub, useRoot } from '../hub'; import { authorFrom, viewerHandle } from '../pages/blog'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts', 'members', 'projects', 'releases'] }; +export const handle = topicsFor('author'); export function loader({ context, params }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/blog-mine.tsx b/site/app/routes/blog-mine.tsx index 96a65f9..4352318 100644 --- a/site/app/routes/blog-mine.tsx +++ b/site/app/routes/blog-mine.tsx @@ -11,8 +11,9 @@ import { useHub, useRoot } from '../hub'; import { checkedForm, EDITOR_PATH, member, viewerHandle } from '../pages/blog'; import { postPath, STATUS_LABEL, type PostStatus } from '../posts'; import { isRefused } from '../profile'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts'] }; +export const handle = topicsFor('blog-mine'); export function loader({ context }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/blog-new.tsx b/site/app/routes/blog-new.tsx index fa475bb..0ce1402 100644 --- a/site/app/routes/blog-new.tsx +++ b/site/app/routes/blog-new.tsx @@ -7,8 +7,9 @@ import { hubContext } from '../context'; import { useRoot } from '../hub'; import { checkedForm, EDITOR_PATH, member } from '../pages/blog'; import { isRefused } from '../profile'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts'] }; +export const handle = topicsFor('blog-new'); export function loader({ context }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/blog.tsx b/site/app/routes/blog.tsx index 9de6e39..aa148fa 100644 --- a/site/app/routes/blog.tsx +++ b/site/app/routes/blog.tsx @@ -12,8 +12,9 @@ import { Page } from '../components/Shell'; import { hubContext } from '../context'; import { useHub, useRoot } from '../hub'; import { viewerHandle } from '../pages/blog'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts', 'members', 'projects'] }; +export const handle = topicsFor('blog'); export function loader({ context }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/changelog.tsx b/site/app/routes/changelog.tsx index 410eb4a..14dfa46 100644 --- a/site/app/routes/changelog.tsx +++ b/site/app/routes/changelog.tsx @@ -5,6 +5,9 @@ import { Page } from '../components/Shell'; import Timeline from '../components/Timeline'; import { day } from '../format'; import { useHub } from '../hub'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('changelog'); export default function Changelog() { const { rootProjects, epics, releases } = useHub(); diff --git a/site/app/routes/editor.tsx b/site/app/routes/editor.tsx index 871fbeb..ce09aff 100644 --- a/site/app/routes/editor.tsx +++ b/site/app/routes/editor.tsx @@ -24,9 +24,10 @@ import { type ShortLink, type UploadKind, } from '../profile'; +import { topicsFor } from '../topics'; import type { Member } from '../types'; -export const handle = { topics: ['profiles', 'members', 'projects'] }; +export const handle = topicsFor('editor'); const SINGLE: BlockType[] = [ 'bio', diff --git a/site/app/routes/epics.tsx b/site/app/routes/epics.tsx index d53910d..4c99c2d 100644 --- a/site/app/routes/epics.tsx +++ b/site/app/routes/epics.tsx @@ -3,7 +3,7 @@ import type { Route } from './+types/epics'; import { accentStyle, Avatar } from '../components/kit/basics'; import { ProjectNode } from '../components/kit/directory'; import { ReleaseTimeline } from '../components/kit/live'; -import { Fact, loadRail, PageHeader, RAIL_TOPICS, StandardRail } from '../components/kit/page'; +import { Fact, loadRail, PageHeader, StandardRail } from '../components/kit/page'; import Status from '../components/Status'; import TagList from '../components/TagList'; import { Page } from '../components/Shell'; @@ -11,9 +11,10 @@ import { hubContext } from '../context'; import { day } from '../format'; import { useHub } from '../hub'; import { useSpotlight } from '../motion/motion'; +import { topicsFor } from '../topics'; import type { Project } from '../types'; -export const handle = { topics: RAIL_TOPICS }; +export const handle = topicsFor('epics'); const WINDOW_DAYS = 180; diff --git a/site/app/routes/home.tsx b/site/app/routes/home.tsx index 21769c5..70b983a 100644 --- a/site/app/routes/home.tsx +++ b/site/app/routes/home.tsx @@ -17,10 +17,9 @@ import { hubContext } from '../context'; import { modelFrom, useHub, useRoot } from '../hub'; import { Field, Mark } from '../motion/canvases'; import { visible } from '../settings'; +import { topicsFor } from '../topics'; -export const handle = { - topics: ['deploys', 'releases', 'projects', 'members', 'stats', 'uptime', 'messages', 'posts'], -}; +export const handle = topicsFor('home'); export function loader({ context }: Route.LoaderArgs) { const { directory, reads, settings, messages } = context.get(hubContext); diff --git a/site/app/routes/kit.tsx b/site/app/routes/kit.tsx index 7b12be1..45acc4e 100644 --- a/site/app/routes/kit.tsx +++ b/site/app/routes/kit.tsx @@ -13,8 +13,11 @@ import Status from '../components/Status'; import { useHub, useRoot } from '../hub'; import { Field, Mark } from '../motion/canvases'; import { COUNTERS, MOTION } from '../settings'; +import { topicsFor } from '../topics'; import type { ConsoleLine } from '../types'; +export const handle = topicsFor('kit'); + export function loader() { if (!import.meta.env.DEV) { throw data(null, { status: 404 }); diff --git a/site/app/routes/member-changelog.tsx b/site/app/routes/member-changelog.tsx index 1b4e0ea..89e5184 100644 --- a/site/app/routes/member-changelog.tsx +++ b/site/app/routes/member-changelog.tsx @@ -7,6 +7,9 @@ import type { Route } from './+types/member-changelog'; import Timeline from '../components/Timeline'; import { found, modelFrom, useHub } from '../hub'; import { changelogPath } from '../model'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('member-changelog'); export function loader({ context, params }: Route.LoaderArgs) { return { handle: found(modelFrom(context).memberByHandle(params.handle)).handle }; diff --git a/site/app/routes/member.tsx b/site/app/routes/member.tsx index a75969b..8207bef 100644 --- a/site/app/routes/member.tsx +++ b/site/app/routes/member.tsx @@ -2,9 +2,10 @@ import { redirect } from 'react-router'; import type { Route } from './+types/member'; import { hubContext } from '../context'; import { found, modelFrom } from '../hub'; -import { loadMember, MEMBER_TOPICS, MemberPage } from '../pages/member-page'; +import { loadMember, MemberPage } from '../pages/member-page'; +import { topicsFor } from '../topics'; -export const handle = { topics: MEMBER_TOPICS }; +export const handle = topicsFor('member'); export function loader({ context, params }: Route.LoaderArgs) { const member = found(modelFrom(context).memberByHandle(params.handle)); diff --git a/site/app/routes/members.tsx b/site/app/routes/members.tsx index 260af4f..18f6938 100644 --- a/site/app/routes/members.tsx +++ b/site/app/routes/members.tsx @@ -2,15 +2,17 @@ import { useCallback, useState, type CSSProperties } from 'react'; import { Link } from 'react-router'; import type { Route } from './+types/members'; import { accentStyle, Avatar, Band } from '../components/kit/basics'; -import { Fact, loadRail, PageHeader, RAIL_TOPICS, StandardRail } from '../components/kit/page'; +import { Fact, loadRail, PageHeader, StandardRail } from '../components/kit/page'; import { Page } from '../components/Shell'; import TagList from '../components/TagList'; import { hubContext } from '../context'; import { useHub } from '../hub'; +import { useOnline } from '../live'; import { useSpotlight, useTilt } from '../motion/motion'; +import { topicsFor } from '../topics'; import type { Member } from '../types'; -export const handle = { topics: RAIL_TOPICS }; +export const handle = topicsFor('members'); export function loader({ context }: Route.LoaderArgs) { return loadRail(context.get(hubContext)); @@ -21,11 +23,13 @@ function Card({ projects, releases, index, + online, }: { member: Member; projects: number; releases: number; index: number; + online: boolean; }) { const tilt = useTilt(); const spot = useSpotlight(); @@ -47,7 +51,7 @@ function Card({ {member.coverUrl && } - +

@@ -100,6 +104,7 @@ export default function Members({ loaderData }: Route.ComponentProps) { const columns = hub.projects.filter((project) => hub.members.some((member) => hub.isCredited(project, member)), ); + const online = useOnline(); return ( }> ))}

diff --git a/site/app/routes/messages.tsx b/site/app/routes/messages.tsx index 6601cc2..0184b05 100644 --- a/site/app/routes/messages.tsx +++ b/site/app/routes/messages.tsx @@ -8,9 +8,10 @@ import { hubContext } from '../context'; import { day } from '../format'; import { useRoot } from '../hub'; import { useSpotlight } from '../motion/motion'; +import { topicsFor } from '../topics'; import type { Message } from '../types'; -export const handle = { topics: ['messages'] }; +export const handle = topicsFor('messages'); const LIMIT = 1000; diff --git a/site/app/routes/not-found.tsx b/site/app/routes/not-found.tsx index c02b408..ab91e77 100644 --- a/site/app/routes/not-found.tsx +++ b/site/app/routes/not-found.tsx @@ -1,5 +1,8 @@ import { data } from 'react-router'; import NotFound from '../components/NotFound'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('not-found'); export function loader() { return data(null, { status: 404 }); diff --git a/site/app/routes/post-editor.tsx b/site/app/routes/post-editor.tsx index b7163d6..8426528 100644 --- a/site/app/routes/post-editor.tsx +++ b/site/app/routes/post-editor.tsx @@ -21,8 +21,9 @@ import { Field } from '../motion/canvases'; import { checkedForm, editablePost, viewerHandle } from '../pages/blog'; import { POST_LIMITS, postPath, slugify, STATUS_LABEL, type PostFields } from '../posts'; import { isRefused, uploadUrl } from '../profile'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts', 'members', 'projects'] }; +export const handle = topicsFor('post-editor'); const PostEditor = lazy(() => import('../editor/PostEditor')); diff --git a/site/app/routes/post.tsx b/site/app/routes/post.tsx index b602b38..d740e6b 100644 --- a/site/app/routes/post.tsx +++ b/site/app/routes/post.tsx @@ -1,6 +1,7 @@ import { data, Link, redirect } from 'react-router'; import type { Route } from './+types/post'; import { accentStyle, Avatar, Button, Cover, Panel } from '../components/kit/basics'; +import { EditingBadge, ReadingBadge } from '../components/kit/live'; import { Fact, PageHeader } from '../components/kit/page'; import { PostRow } from '../components/kit/posts'; import { WindowBar } from '../components/kit/projects'; @@ -9,12 +10,17 @@ import { Page } from '../components/Shell'; import TagList from '../components/TagList'; import { hubContext } from '../context'; import { day } from '../format'; -import { useHub } from '../hub'; +import { useHub, useRoot } from '../hub'; +import { useView, useViews } from '../live'; import { createModel } from '../model'; import { accessTo, authorFrom, EDITOR_PATH } from '../pages/blog'; import { uploadUrl } from '../profile'; +import { topicsFor, type KeyedMatch } from '../topics'; -export const handle = { topics: ['posts', 'members', 'projects'] }; +const postId = ({ loaderData }: KeyedMatch) => + (loaderData as { post?: { id: string } } | undefined)?.post?.id; + +export const handle = topicsFor('post', { reading: postId, editing: postId }); const HEADING = /]*>([\s\S]*?)<\/h\1>/g; @@ -35,6 +41,10 @@ export function loader({ context, params }: Route.LoaderArgs) { const mine = model.postsBy(author); const index = mine.findIndex((entry) => entry.id === post.id); return { + reading: hub.presence.reading(post.id), + editing: hub.collab + .editing(post.id) + .map(({ handle, name, color }) => ({ handle, name, color })), post: { id: post.id, slug: post.slug, @@ -74,6 +84,8 @@ export default function Post({ loaderData }: Route.ComponentProps) { const owner = project ? hub.memberByHandle(project.owner) : undefined; const more = hub.postsBy(author).filter((entry) => entry.id !== post.id); const draft = post.status !== 'published'; + const views = useViews(useRoot()!.views)('post', post.id); + useView('post', post.id); return ( } title={post.title} + aside={ + <> + + + + } subtitle={post.summary} actions={ editable && ( @@ -185,6 +203,7 @@ export default function Post({ loaderData }: Route.ComponentProps) { /> )} + {views > 0 && } } > diff --git a/site/app/routes/privacy.tsx b/site/app/routes/privacy.tsx index f190e5b..a0a7a85 100644 --- a/site/app/routes/privacy.tsx +++ b/site/app/routes/privacy.tsx @@ -1,5 +1,8 @@ import { Fact, PageHeader } from '../components/kit/page'; import { Page } from '../components/Shell'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('privacy'); const KEPT = [ { @@ -9,7 +12,22 @@ const KEPT = [ }, { what: 'Addresses', - how: 'Used only to limit how often a sign-in can be tried.', + how: 'Used only to limit how often a sign-in, a message, a subscription or a view can be sent.', + kept: 'up to 1 hour', + }, + { + what: 'Views', + how: 'A count per post and project, one a day per visitor. The visitor is a hash of the address, the browser and a daily random value; the hash is never stored and cannot be reversed.', + kept: 'the counts for ever, the daily check for 2 days', + }, + { + what: 'Reading now', + how: 'How many open tabs are on the site and on each post or project, counted by a random tab id, never by address.', + kept: '90 seconds after a tab closes', + }, + { + what: 'Visitor map', + how: 'The country a page was opened from, as a count per country from the edge, for the map on /status. Never an address or a city.', kept: '10 minutes', }, { diff --git a/site/app/routes/project-changelog.tsx b/site/app/routes/project-changelog.tsx index d1d64b3..4d972ff 100644 --- a/site/app/routes/project-changelog.tsx +++ b/site/app/routes/project-changelog.tsx @@ -10,6 +10,9 @@ import Timeline from '../components/Timeline'; import { day } from '../format'; import { found, modelFrom, useHub } from '../hub'; import { changelogPath } from '../model'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('project-changelog'); export function loader({ context, params }: Route.LoaderArgs) { const project = found(modelFrom(context).projectByRoute(params.parentSlug, params.slug)); diff --git a/site/app/routes/projects.tsx b/site/app/routes/projects.tsx index 6655ca6..1c52449 100644 --- a/site/app/routes/projects.tsx +++ b/site/app/routes/projects.tsx @@ -3,7 +3,7 @@ import { Link } from 'react-router'; import type { Route } from './+types/projects'; import { accentStyle, Avatar, Cover } from '../components/kit/basics'; import { Chips, Pulse } from '../components/kit/directory'; -import { Fact, loadRail, PageHeader, RAIL_TOPICS, StandardRail } from '../components/kit/page'; +import { Fact, loadRail, PageHeader, StandardRail } from '../components/kit/page'; import Status from '../components/Status'; import { Page } from '../components/Shell'; import { hubContext } from '../context'; @@ -11,9 +11,10 @@ import { day } from '../format'; import { useHub } from '../hub'; import { STATUS_LABEL } from '../model'; import { useSpotlight, useTilt } from '../motion/motion'; +import { topicsFor } from '../topics'; import type { Member, Project, ProjectStatus, Release } from '../types'; -export const handle = { topics: RAIL_TOPICS }; +export const handle = topicsFor('projects'); export function loader({ context }: Route.LoaderArgs) { return { ...loadRail(context.get(hubContext)), today: new Date().toISOString().slice(0, 10) }; diff --git a/site/app/routes/resolved.tsx b/site/app/routes/resolved.tsx index 7534e3a..5c9daa1 100644 --- a/site/app/routes/resolved.tsx +++ b/site/app/routes/resolved.tsx @@ -2,10 +2,16 @@ import { data } from 'react-router'; import type { Route } from './+types/resolved'; import { hubContext } from '../context'; import { modelFrom } from '../hub'; -import { loadMember, MEMBER_TOPICS, MemberPage } from '../pages/member-page'; -import { loadProject, PROJECT_TOPICS, ProjectPage } from '../pages/project-page'; +import { loadMember, MemberPage } from '../pages/member-page'; +import { loadProject, ProjectPage } from '../pages/project-page'; +import { topicsFor } from '../topics'; -export const handle = { topics: [...new Set([...PROJECT_TOPICS, ...MEMBER_TOPICS])] }; +export const handle = topicsFor('resolved', { + reading: ({ loaderData }) => { + const loaded = loaderData as { page?: string; slug?: string } | undefined; + return loaded?.page === 'project' ? loaded.slug : undefined; + }, +}); export function loader({ context, params }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/search.tsx b/site/app/routes/search.tsx index 4511e42..5c01937 100644 --- a/site/app/routes/search.tsx +++ b/site/app/routes/search.tsx @@ -11,6 +11,9 @@ import { day } from '../format'; import { useHub, useRoot } from '../hub'; import { changelogPath } from '../model'; import { search, type Hit } from '../search'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('search'); const KINDS: Hit['kind'][] = ['member', 'project', 'release']; diff --git a/site/app/routes/sign-in.tsx b/site/app/routes/sign-in.tsx index 225bfa7..107d735 100644 --- a/site/app/routes/sign-in.tsx +++ b/site/app/routes/sign-in.tsx @@ -4,6 +4,9 @@ import { Page } from '../components/Shell'; import { useRoot } from '../hub'; import { Field, Mark } from '../motion/canvases'; import { useMagnetic } from '../motion/motion'; +import { topicsFor } from '../topics'; + +export const handle = topicsFor('sign-in'); export default function SignIn() { const root = useRoot(); diff --git a/site/app/routes/status.tsx b/site/app/routes/status.tsx index 9b3b0a4..3f9a950 100644 --- a/site/app/routes/status.tsx +++ b/site/app/routes/status.tsx @@ -2,6 +2,7 @@ import { useState } from 'react'; import { Link } from 'react-router'; import type { Route } from './+types/status'; import { Console } from '../components/kit/live'; +import { VisitorMap } from '../components/kit/map'; import { Fact, PageHeader } from '../components/kit/page'; import { Page } from '../components/Shell'; import Status from '../components/Status'; @@ -9,21 +10,23 @@ import { hubContext } from '../context'; import { day, shortTag } from '../format'; import { useRoot } from '../hub'; import { useLiveMessages } from '../live'; +import { topicsFor } from '../topics'; import type { Day } from '../types'; const DAYS = 30; const DAY_MS = 86_400_000; -export const handle = { topics: ['deploys', 'uptime', 'ping', 'releases', 'projects'] }; +export const handle = topicsFor('status'); export function loader({ context }: Route.LoaderArgs) { - const { reads, directory, settings } = context.get(hubContext); + const { reads, directory, settings, presence } = context.get(hubContext); return { running: reads.running(directory), up: reads.upNow(), uptime: reads.uptime(DAYS), lines: reads.consoleLines(settings['rail.console.lines'] * 3), today: new Date().toISOString().slice(0, 10), + countries: presence.countries(), }; } @@ -50,7 +53,7 @@ function Strip({ days, today }: { days: Day[]; today: string }) { } export default function StatusPage({ loaderData }: Route.ComponentProps) { - const { running, up, uptime, lines, today } = loaderData; + const { running, up, uptime, lines, today, countries } = loaderData; const limit = useRoot()!.settings['rail.console.lines'] * 3; const live = running.filter((project) => up.includes(project.slug)).length; const [pinged, setPinged] = useState>({}); @@ -130,6 +133,7 @@ export default function StatusPage({ loaderData }: Route.ComponentProps) { })} + ); diff --git a/site/app/routes/subscribe.tsx b/site/app/routes/subscribe.tsx index 4b352b3..186c212 100644 --- a/site/app/routes/subscribe.tsx +++ b/site/app/routes/subscribe.tsx @@ -10,8 +10,9 @@ import { Page } from '../components/Shell'; import { hubContext } from '../context'; import { useHub } from '../hub'; import { isCadence, knownScope, parseScope, type Cadence, type Scope } from '../subscribe'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['posts', 'members', 'projects'] }; +export const handle = topicsFor('subscribe'); export function loader({ context }: Route.LoaderArgs) { const hub = context.get(hubContext); diff --git a/site/app/routes/subscription.tsx b/site/app/routes/subscription.tsx index f24d9bf..f057dfc 100644 --- a/site/app/routes/subscription.tsx +++ b/site/app/routes/subscription.tsx @@ -10,8 +10,9 @@ import { hubContext, type Hub } from '../context'; import { day } from '../format'; import { useHub } from '../hub'; import { describeChoice, parseScope, scopeKey, type Cadence, type Scope } from '../subscribe'; +import { topicsFor } from '../topics'; -export const handle = { topics: ['members', 'projects'] }; +export const handle = topicsFor('subscription'); function verified(hub: Hub, params: Route.LoaderArgs['params']) { const subscriber = hub.subscribers.verify(params.purpose, params.id, params.sig); diff --git a/site/app/topics.ts b/site/app/topics.ts new file mode 100644 index 0000000..877551d --- /dev/null +++ b/site/app/topics.ts @@ -0,0 +1,112 @@ +export const TOPICS = [ + 'members', + 'profiles', + 'projects', + 'releases', + 'settings', + 'posts', + 'sessions', + 'deploys', + 'uptime', + 'ping', + 'stats', + 'messages', + 'errors', + 'subscribers', + 'console', + 'presence', + 'reading', + 'editing', + 'views', + 'map', +] as const; + +export type Topic = (typeof TOPICS)[number]; + +export const MAX_TOPICS = 12; + +export const PATCHED: ReadonlySet = new Set([ + 'presence', + 'reading', + 'editing', + 'views', + 'ping', + 'map', +]); + +export function revalidates(topic: string) { + return !PATCHED.has(topic as Topic); +} + +const DIRECTORY: Topic[] = ['members', 'profiles', 'projects', 'releases', 'settings', 'posts']; +export const ROOT_TOPICS: Topic[] = [...DIRECTORY, 'presence', 'views']; +const RAIL: Topic[] = ['deploys', 'uptime']; + +export const ROUTE_TOPICS = { + home: [...ROOT_TOPICS, ...RAIL, 'stats', 'messages'], + members: [...ROOT_TOPICS, ...RAIL], + member: [...ROOT_TOPICS, ...RAIL], + epics: [...ROOT_TOPICS, ...RAIL], + projects: [...ROOT_TOPICS, ...RAIL], + resolved: [...ROOT_TOPICS, ...RAIL, 'reading'], + changelog: ROOT_TOPICS, + 'member-changelog': ROOT_TOPICS, + 'project-changelog': ROOT_TOPICS, + status: [...ROOT_TOPICS, ...RAIL, 'ping', 'map'], + admin: [...ROOT_TOPICS, 'errors', 'subscribers'], + messages: [...ROOT_TOPICS, 'messages'], + blog: ROOT_TOPICS, + author: ROOT_TOPICS, + post: [...ROOT_TOPICS, 'reading', 'editing'], + 'blog-new': ROOT_TOPICS, + 'blog-mine': ROOT_TOPICS, + 'post-editor': ROOT_TOPICS, + editor: ROOT_TOPICS, + subscribe: ROOT_TOPICS, + subscription: ROOT_TOPICS, + search: ROOT_TOPICS, + privacy: ROOT_TOPICS, + 'sign-in': ROOT_TOPICS, + kit: ROOT_TOPICS, + 'not-found': ROOT_TOPICS, +} satisfies Record; + +export type RouteName = keyof typeof ROUTE_TOPICS; + +export interface KeyedMatch { + loaderData: unknown; + params: Record; +} + +export type Keys = Partial string | undefined>>; + +export interface LiveHandle { + topics: Topic[]; + keys?: Keys; +} + +export function topicsFor(route: RouteName, keys?: Keys): LiveHandle { + return keys ? { topics: ROUTE_TOPICS[route], keys } : { topics: ROUTE_TOPICS[route] }; +} + +export function subscription(handles: LiveHandle[], matches: KeyedMatch[]) { + const topics = new Set(); + let reading: string | undefined; + handles.forEach((handle, index) => { + for (const topic of handle.topics) { + const keyed = handle.keys?.[topic]; + if (!keyed) { + topics.add(topic); + continue; + } + const key = keyed(matches[index]); + if (key) { + topics.add(`${topic}:${key}`); + if (topic === 'reading') { + reading = key; + } + } + } + }); + return { topics: [...topics].sort(), reading }; +} diff --git a/site/app/types.ts b/site/app/types.ts index ae882c1..47a4b7e 100644 --- a/site/app/types.ts +++ b/site/app/types.ts @@ -154,3 +154,13 @@ export interface Viewer { avatarUrl: string; owner: boolean; } + +export interface Presence { + here: number; + online: string[]; +} + +export interface Country { + code: string; + count: number; +} diff --git a/site/server/address.ts b/site/server/address.ts index ac357b8..2012b89 100644 --- a/site/server/address.ts +++ b/site/server/address.ts @@ -38,11 +38,25 @@ export function viaEdge(request: IncomingMessage) { return request.headers['x-forwarded-for'] !== undefined; } -export function clientAddress(request: IncomingMessage, ranges = cloudflare) { +function peerOf(request: IncomingMessage) { const forwarded = (request.headers['x-forwarded-for'] as string | undefined) ?.split(',') .map((part) => part.trim()); - const peer = plain(forwarded?.at(-1) ?? request.socket.remoteAddress ?? ''); + return { forwarded, peer: plain(forwarded?.at(-1) ?? request.socket.remoteAddress ?? '') }; +} + +export function fromCloudflare(request: IncomingMessage, ranges = cloudflare) { + const { forwarded, peer } = peerOf(request); + return forwarded !== undefined && inRanges(ranges, peer); +} + +export function clientAddress(request: IncomingMessage, ranges = cloudflare) { + const { peer } = peerOf(request); const original = request.headers['cf-connecting-ip'] as string | undefined; - return forwarded && original && inRanges(ranges, peer) ? plain(original) : peer; + return original && fromCloudflare(request, ranges) ? plain(original) : peer; +} + +export function countryOf(request: IncomingMessage, trusted = fromCloudflare(request)) { + const code = String(request.headers['cf-ipcountry'] ?? '').toUpperCase(); + return trusted && /^[A-Z]{2}$/.test(code) && !['XX', 'T1'].includes(code) ? code : undefined; } diff --git a/site/server/admin.ts b/site/server/admin.ts index 4073101..5527df1 100644 --- a/site/server/admin.ts +++ b/site/server/admin.ts @@ -57,6 +57,7 @@ export function createAdmin(db: Database, bus: Bus, settings: Settings, sessions async signOutEveryone(by: string) { const removed = await sessions.revokeEveryone(); + const at = new Date().toISOString(); await bus.record({ topic: 'console', kind: 'sessions', @@ -64,7 +65,16 @@ export function createAdmin(db: Database, bus: Bus, settings: Settings, sessions actor: by, summary: `Everyone signed out: ${removed} sessions ended`, detail: { removed }, - at: new Date().toISOString(), + at, + }); + await bus.publish({ + topic: 'sessions', + kind: 'revoked', + public: false, + actor: by, + summary: 'Every session ended', + detail: { removed }, + at, }); return removed; }, diff --git a/site/server/app.ts b/site/server/app.ts index 26ee954..4b15aea 100644 --- a/site/server/app.ts +++ b/site/server/app.ts @@ -18,9 +18,11 @@ import type { createMetrics } from './metrics.ts'; import { slidingLimit, type Redis } from './redis.ts'; import { clientAddress } from './address.ts'; import type { Admin } from './admin.ts'; +import { sameOrigin } from './auth.ts'; import type { Messages } from './messages.ts'; import type { Collab } from './collab.ts'; import type { Posts } from './posts.ts'; +import type { Presence } from './presence.ts'; import type { Presentations } from './presentation.ts'; import type { Profiles } from './profiles.ts'; import type { Writeback } from './writeback.ts'; @@ -31,6 +33,7 @@ import type { Settings } from './settings.ts'; import type { Snapshot } from './snapshot.ts'; import type { Subscribers } from './subscribers.ts'; import type { Sync } from './sync.ts'; +import type { Views } from './views.ts'; export interface Services { db: Database; @@ -49,6 +52,8 @@ export interface Services { posts: Posts; subscribers: Subscribers; collab: Collab; + presence: Presence; + views: Views; auth: Auth; live: Live; sync: Sync; @@ -202,6 +207,17 @@ export function createApp( app.use(auth.router); app.get('/live', live.handler); + app.post('/views/:kind/:key', async (request, response) => { + if (sameOrigin(request)) { + await services.views.count({ + kind: request.params.kind, + key: request.params.key, + address: clientAddress(request), + userAgent: String(request.headers['user-agent'] ?? ''), + }); + } + response.status(204).end(); + }); app.get('/admin/email', async (request, response) => { const signedIn = await auth.current(request, response); const kind = String(request.query.kind ?? ''); @@ -268,6 +284,8 @@ export function createApp( posts: services.posts, subscribers: services.subscribers, collab: services.collab, + presence: services.presence, + views: services.views, address: clientAddress(request), origin: new URL(config.appUrl).origin, viewer: signedIn diff --git a/site/server/auth.test.ts b/site/server/auth.test.ts index 11ab32a..fc103ac 100644 --- a/site/server/auth.test.ts +++ b/site/server/auth.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import type { IncomingMessage } from 'node:http'; import { describe, it } from 'node:test'; -import { clientAddress, cloudflareRanges } from './address.ts'; +import { clientAddress, cloudflareRanges, countryOf, fromCloudflare } from './address.ts'; import { cookies, safeReturn, sameOrigin } from './auth.ts'; import { createVerifier, sign } from './internal.ts'; import type { Redis } from './redis.ts'; @@ -98,6 +98,19 @@ describe('clientAddress', () => { '172.18.0.9', ); }); + + it('reads the country only from Cloudflare, and never an unknown code', () => { + const edge = { 'x-forwarded-for': '173.245.48.10', 'cf-ipcountry': 'de' }; + assert.ok(fromCloudflare(request(edge), ranges)); + assert.equal(countryOf(request(edge), fromCloudflare(request(edge), ranges)), 'DE'); + assert.equal( + countryOf(request({ 'x-forwarded-for': '203.0.113.9', 'cf-ipcountry': 'DE' })), + undefined, + ); + assert.equal(countryOf(request({ ...edge, 'cf-ipcountry': 'XX' }), true), undefined); + assert.equal(countryOf(request({ ...edge, 'cf-ipcountry': 'T1' }), true), undefined); + assert.equal(countryOf(request({ 'x-forwarded-for': '173.245.48.10' }), true), undefined); + }); }); describe('internal request verification', () => { diff --git a/site/server/collab.ts b/site/server/collab.ts index 2c803dc..91305a3 100644 --- a/site/server/collab.ts +++ b/site/server/collab.ts @@ -3,6 +3,7 @@ import type { Duplex } from 'node:stream'; import { Server } from '@hocuspocus/server'; import * as Y from 'yjs'; import { cookies } from './auth.ts'; +import type { Bus } from './bus.ts'; import { config } from './config.ts'; import type { Database } from './db.ts'; import { log } from './log.ts'; @@ -20,10 +21,62 @@ export interface Collaborator { color: string; } -export function createCollab(db: Database, posts: Posts, sessions: Sessions) { +export function createCollab(db: Database, posts: Posts, sessions: Sessions, bus: Bus) { const member = db.prepare( `SELECT m.handle, m.name, p.accent FROM members m LEFT JOIN profiles p USING (handle) WHERE m.login = ? COLLATE NOCASE`, ); + const announced = new Map(); + + function editing(id: string) { + const document = server.hocuspocus.documents.get(id); + const seen = new Map(); + for (const state of document?.awareness.getStates().values() ?? []) { + const user = (state as { user?: Collaborator }).user; + if (user?.handle) { + seen.set(user.handle, user); + } + } + return [...seen.values()]; + } + + async function announce(id: string) { + const editors = editing(id).map(({ handle, name, color }) => ({ handle, name, color })); + const signature = editors.map((editor) => editor.handle).join(','); + if ((announced.get(id) ?? '') === signature) { + return; + } + if (signature) { + announced.set(id, signature); + } else { + announced.delete(id); + } + await bus.publish({ + topic: 'editing', + key: id, + kind: 'editing', + public: true, + summary: `${editors.length} editing ${id}`, + detail: { id, editors }, + at: new Date().toISOString(), + }); + } + + bus.listen(({ event }) => { + if (event.topic !== 'sessions') { + return; + } + if (!event.key) { + server.hocuspocus.closeConnections(); + return; + } + for (const document of server.hocuspocus.documents.values()) { + for (const connection of document.getConnections()) { + if ((connection.context as Collaborator).login.toLowerCase() === event.key) { + connection.close(); + } + } + } + }); const server = new Server({ quiet: true, @@ -88,6 +141,12 @@ export function createCollab(db: Database, posts: Posts, sessions: Sessions) { } } }, + async onAwarenessUpdate({ documentName }) { + await announce(documentName); + }, + async onDisconnect({ documentName }) { + await announce(documentName); + }, }); const upgrades = server as unknown as { @@ -118,17 +177,7 @@ export function createCollab(db: Database, posts: Posts, sessions: Sessions) { await connection.disconnect(); } }, - editing(id: string) { - const document = server.hocuspocus.documents.get(id); - const seen = new Map(); - for (const state of document?.awareness.getStates().values() ?? []) { - const user = (state as { user?: Collaborator }).user; - if (user?.handle) { - seen.set(user.handle, user); - } - } - return [...seen.values()]; - }, + editing, destroy() { server.hocuspocus.closeConnections(); }, diff --git a/site/server/live-audit.test.ts b/site/server/live-audit.test.ts new file mode 100644 index 0000000..ddbfdea --- /dev/null +++ b/site/server/live-audit.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { describe, it, mock } from 'node:test'; +import { + MAX_TOPICS, + revalidates, + ROOT_TOPICS, + ROUTE_TOPICS, + subscription, + TOPICS, + topicsFor, + type RouteName, + type Topic, +} from '../app/topics.ts'; +import type { HubEvent } from './bus.ts'; +import { createClient, parseTopics } from './live.ts'; + +const DIRECTORY: Topic[] = ['members', 'profiles', 'projects', 'releases', 'settings', 'posts']; +const EVERY_PAGE: Topic[] = [...DIRECTORY, 'presence', 'views']; +const RAIL: Topic[] = ['deploys', 'uptime']; + +const DEPENDS: Record = { + home: [...EVERY_PAGE, ...RAIL, 'stats', 'messages'], + members: [...EVERY_PAGE, ...RAIL], + member: [...EVERY_PAGE, ...RAIL], + epics: [...EVERY_PAGE, ...RAIL], + projects: [...EVERY_PAGE, ...RAIL], + resolved: [...EVERY_PAGE, ...RAIL, 'reading'], + changelog: EVERY_PAGE, + 'member-changelog': EVERY_PAGE, + 'project-changelog': EVERY_PAGE, + status: [...EVERY_PAGE, ...RAIL, 'ping', 'map'], + admin: [...EVERY_PAGE, 'errors', 'subscribers'], + messages: [...EVERY_PAGE, 'messages'], + blog: EVERY_PAGE, + author: EVERY_PAGE, + post: [...EVERY_PAGE, 'reading', 'editing'], + 'blog-new': EVERY_PAGE, + 'blog-mine': EVERY_PAGE, + 'post-editor': EVERY_PAGE, + editor: EVERY_PAGE, + subscribe: EVERY_PAGE, + subscription: EVERY_PAGE, + search: EVERY_PAGE, + privacy: EVERY_PAGE, + 'sign-in': EVERY_PAGE, + kit: EVERY_PAGE, + 'not-found': EVERY_PAGE, +}; + +const KEY = 'sample-key'; + +const event = (topic: Topic): HubEvent => ({ + topic, + key: KEY, + kind: topic, + public: true, + summary: topic, + detail: {}, + at: '2026-11-07T00:00:00.000Z', +}); + +function subscribed(route: RouteName) { + const keyed = () => KEY; + return subscription( + [{ topics: ROOT_TOPICS }, topicsFor(route, { reading: keyed, editing: keyed })], + [ + { loaderData: undefined, params: {} }, + { loaderData: undefined, params: {} }, + ], + ).topics; +} + +const source = (file: string) => readFileSync(new URL(`../app/${file}`, import.meta.url), 'utf8'); + +describe('live audit', () => { + const routes = [...source('routes.ts').matchAll(/'routes\/([a-z-]+)\.tsx'/g)].map( + ([, name]) => name, + ); + + it('maps every component route and nothing else', () => { + assert.deepEqual([...new Set(routes)].sort(), Object.keys(ROUTE_TOPICS).sort()); + }); + + it('every route module declares its handle from the map', () => { + for (const route of new Set(routes)) { + assert.match( + source(`routes/${route}.tsx`), + new RegExp(`export const handle = topicsFor\\('${route}'`), + `${route} does not use topicsFor`, + ); + } + assert.match(source('root.tsx'), /handle: LiveHandle = \{ topics: ROOT_TOPICS \}/); + }); + + it('every route stays within the stream cap and the server accepts its subscription', () => { + for (const route of Object.keys(ROUTE_TOPICS) as RouteName[]) { + const topics = subscribed(route); + assert.ok(topics.length <= MAX_TOPICS, `${route} asks for ${topics.length} topics`); + assert.deepEqual(parseTopics(topics.join(',')), topics); + } + }); + + it('firing each topic reaches exactly the routes that depend on it', () => { + mock.timers.enable({ apis: ['setTimeout'] }); + for (const topic of TOPICS) { + for (const route of Object.keys(ROUTE_TOPICS) as RouteName[]) { + const frames: string[] = []; + const client = createClient( + (chunk) => frames.push(chunk), + subscribed(route), + false, + undefined, + ); + client.offer({ id: '1-0', event: event(topic) }); + mock.timers.tick(100); + const expected = DEPENDS[route].includes(topic); + assert.equal( + frames.length, + expected ? 1 : 0, + `${topic} ${expected ? 'should reach' : 'should not reach'} ${route}`, + ); + } + } + mock.timers.reset(); + }); + + it('sessions reach no route by subscription; the stream pushes them to members', () => { + for (const route of Object.keys(ROUTE_TOPICS) as RouteName[]) { + assert.ok(!DEPENDS[route].includes('sessions')); + } + }); + + it('patched topics change the DOM from their payload; the rest revalidate the loaders', () => { + for (const topic of ['presence', 'reading', 'editing', 'views', 'ping', 'map']) { + assert.ok(!revalidates(topic), `${topic} must patch in place`); + } + for (const topic of TOPICS.filter( + (entry) => !['presence', 'reading', 'editing', 'views', 'ping', 'map'].includes(entry), + )) { + assert.ok(revalidates(topic), `${topic} must revalidate`); + } + }); +}); diff --git a/site/server/live.test.ts b/site/server/live.test.ts index 434a3a8..8deac05 100644 --- a/site/server/live.test.ts +++ b/site/server/live.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { describe, it, mock } from 'node:test'; import type { HubEvent } from './bus.ts'; -import { createClient, matches, newer, parseTopics } from './live.ts'; +import { createClient, matches, newer, parseKey, parseTopics, payload, signedOut } from './live.ts'; const event = (topic: string, key?: string, isPublic = true): HubEvent => ({ topic, @@ -99,4 +99,49 @@ describe('createClient', () => { assert.ok(!visitor.frames[0].includes('host')); assert.equal(member.frames.length, 2); }); + + it('pushes a frame past the topic filter and flushes it at once', () => { + mock.timers.enable({ apis: ['setTimeout'] }); + const { frames, client: live } = client(false); + live.push({ event: { ...event('sessions'), public: false } }); + assert.equal(frames.length, 0); + live.flush(); + assert.equal(frames.length, 1); + mock.timers.tick(100); + mock.timers.reset(); + assert.equal(frames.length, 1); + }); +}); + +describe('payload', () => { + it('carries the detail of patched topics to everyone and keeps the rest to members', () => { + const presence = { ...event('presence'), detail: { here: 3, online: ['amrmscll'] } }; + assert.deepEqual(payload(presence, false).data, { here: 3, online: ['amrmscll'] }); + const release = { ...event('releases'), detail: { project: 'strata', host: 'x' } }; + assert.ok(!('data' in payload(release, false))); + assert.ok(!('event' in payload(release, false))); + assert.deepEqual((payload(release, true) as { event?: HubEvent }).event, release); + }); +}); + +describe('signedOut', () => { + it('ends every member stream, or only that login, and never a visitor', () => { + const everyone = { ...event('sessions'), key: undefined, public: false }; + const one = { ...event('sessions'), key: 'amrmscll', public: false }; + assert.ok(signedOut({ member: true, login: 'AmrMsCLL' }, everyone)); + assert.ok(signedOut({ member: true, login: 'AmrMsCLL' }, one)); + assert.ok(!signedOut({ member: true, login: 'Omar-Mohamed24' }, one)); + assert.ok(!signedOut({ member: false }, everyone)); + assert.ok(!signedOut({ member: true, login: 'AmrMsCLL' }, event('releases'))); + }); +}); + +describe('parseKey', () => { + it('accepts a slug or id and refuses anything else', () => { + assert.equal(parseKey('strata'), 'strata'); + assert.equal(parseKey('0a1b2c3d4e5f6a7b'), '0a1b2c3d4e5f6a7b'); + assert.equal(parseKey('Strata'), undefined); + assert.equal(parseKey(['a']), undefined); + assert.equal(parseKey(''), undefined); + }); }); diff --git a/site/server/live.ts b/site/server/live.ts index 5422181..ab05e25 100644 --- a/site/server/live.ts +++ b/site/server/live.ts @@ -1,10 +1,13 @@ +import { randomBytes } from 'node:crypto'; import type { Request, Response } from 'express'; +import { MAX_TOPICS, PATCHED, type Topic } from '../app/topics.ts'; import type { Auth } from './auth.ts'; import type { Bus, Delivery, HubEvent } from './bus.ts'; import { toPublicEvent } from './console/public.ts'; +import type { Presence } from './presence.ts'; -const MAX_TOPICS = 12; const TOPIC = /^[a-z]+(?::[a-z0-9-]+)?$/; +const KEY = /^[a-z0-9-]{1,80}$/; const HEARTBEAT_MS = 25_000; const COALESCE_MS = 100; @@ -23,6 +26,10 @@ export function parseTopics(value: unknown) { : undefined; } +export function parseKey(value: unknown) { + return typeof value === 'string' && KEY.test(value) ? value : undefined; +} + export function matches(topics: string[], event: HubEvent) { return topics.some((topic) => topic === event.topic || topic === `${event.topic}:${event.key}`); } @@ -42,6 +49,7 @@ export function payload(event: HubEvent, member: boolean) { key: event.key, kind: event.kind, console: toPublicEvent(event), + ...(PATCHED.has(event.topic as Topic) ? { data: event.detail } : {}), }; return member ? { ...shared, event } : shared; } @@ -57,6 +65,7 @@ export function createClient( const pending = new Map(); function flush() { + clearTimeout(timer); timer = undefined; for (const { id, event } of pending.values()) { write(`${id ? `id: ${id}\n` : ''}data: ${JSON.stringify(payload(event, member))}\n\n`); @@ -64,29 +73,70 @@ export function createClient( pending.clear(); } + function queue(delivery: Delivery) { + const { id, event } = delivery; + if (id) { + last = id; + } + pending.set(`${event.topic}:${event.key ?? ''}:${event.kind}`, delivery); + timer ??= setTimeout(flush, COALESCE_MS); + } + return { offer(delivery: Delivery) { const { id, event } = delivery; if (!matches(topics, event) || (!event.public && !member) || (id && !newer(id, last))) { return; } - if (id) { - last = id; - } - pending.set(`${event.topic}:${event.key ?? ''}:${event.kind}`, delivery); - timer ??= setTimeout(flush, COALESCE_MS); + queue(delivery); }, + push: queue, + flush, close() { clearTimeout(timer); }, }; } -export function createLive(bus: Bus, auth: Auth) { - const clients = new Set, 'offer'>>(); +interface Stream { + offer(delivery: Delivery): void; + push(delivery: Delivery): void; + member: boolean; + login?: string; + end(): void; +} + +export function signedOut(stream: Pick, event: HubEvent) { + return ( + event.topic === 'sessions' && + stream.member && + (!event.key || stream.login?.toLowerCase() === event.key) + ); +} + +export function createLive( + bus: Bus, + auth: Auth, + presence: Presence, + deps: { + handleOf: (login: string) => string | undefined; + countryOf: (request: Request) => string | undefined; + }, +) { + const streams = new Set(); + + function deliver(stream: Stream, delivery: Delivery) { + if (signedOut(stream, delivery.event)) { + stream.push(delivery); + stream.end(); + } else { + stream.offer(delivery); + } + } + bus.listen((delivery) => { - for (const client of clients) { - client.offer(delivery); + for (const stream of streams) { + deliver(stream, delivery); } }); @@ -96,7 +146,7 @@ export function createLive(bus: Bus, auth: Auth) { response.status(400).type('text').send(`Ask for 1 to ${MAX_TOPICS} topics.`); return; } - const member = Boolean(await auth.current(request, response)); + const signedIn = await auth.current(request, response); const header = String(request.headers['last-event-id'] ?? ''); const lastId = /^\d+-\d+$/.test(header) ? header : undefined; response.writeHead(200, { @@ -106,33 +156,58 @@ export function createLive(bus: Bus, auth: Auth) { 'X-Accel-Buffering': 'no', }); response.write('retry: 3000\n\n'); + const tab = randomBytes(6).toString('base64url'); const held: Delivery[] = []; - const holder = { offer: (delivery: Delivery) => held.push(delivery) }; - const client = createClient((chunk) => response.write(chunk), topics, member, lastId); - const heartbeat = setInterval(() => response.write(': ping\n\n'), HEARTBEAT_MS); + const client = createClient( + (chunk) => response.write(chunk), + topics, + Boolean(signedIn), + lastId, + ); + const stream: Stream = { + offer: client.offer, + push: client.push, + member: Boolean(signedIn), + login: signedIn?.session.login, + end() { + client.flush(); + response.end(); + }, + }; + const holder: Stream = { ...stream, offer: (delivery) => held.push(delivery) }; + const heartbeat = setInterval(() => { + response.write(': ping\n\n'); + void presence.touch(tab); + }, HEARTBEAT_MS); let open = true; request.on('close', () => { open = false; clearInterval(heartbeat); client.close(); - clients.delete(holder); - clients.delete(client); + streams.delete(holder); + streams.delete(stream); + void presence.leave(tab); }); - clients.add(holder); + streams.add(holder); const replay = lastId ? await bus.since(lastId) : []; - clients.delete(holder); + streams.delete(holder); if (open) { for (const delivery of [...replay, ...held]) { - client.offer(delivery); + deliver(stream, delivery); } - clients.add(client); + streams.add(stream); + await presence.join(tab, { + reading: parseKey(request.query.reading), + handle: signedIn && deps.handleOf(signedIn.session.login), + country: deps.countryOf(request), + }); } } return { handler, get connections() { - return clients.size; + return streams.size; }, }; } diff --git a/site/server/main.ts b/site/server/main.ts index aea0fc5..a570c20 100644 --- a/site/server/main.ts +++ b/site/server/main.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import type { RequestHandler } from 'express'; import type { ServerBuild } from 'react-router'; -import { refreshCloudflareRanges } from './address.ts'; +import { countryOf, refreshCloudflareRanges } from './address.ts'; import { createAdmin } from './admin.ts'; import { createApp } from './app.ts'; import { createMessages } from './messages.ts'; @@ -22,6 +22,7 @@ import { createMetrics } from './metrics.ts'; import { connectRedis, slidingLimit } from './redis.ts'; import { createSessions } from './sessions.ts'; import { github } from './github.ts'; +import { createPresence } from './presence.ts'; import { createPresentations } from './presentation.ts'; import { createProfiles } from './profiles.ts'; import { createWriteback } from './writeback.ts'; @@ -32,6 +33,7 @@ import { createSnapshot } from './snapshot.ts'; import { createSubscribers } from './subscribers.ts'; import { createSync } from './sync.ts'; import { pingAll } from './uptime.ts'; +import { createViews } from './views.ts'; const MINUTE = 60_000; const HOUR = 60 * MINUTE; @@ -45,11 +47,24 @@ await bus.ready; const capture = createErrors(db, bus, alertOwner); const sessions = createSessions(redis, config.sessionSecret); const auth = createAuth(sessions, redis); -const live = createLive(bus, auth); +const presence = createPresence(redis, bus); +const live = createLive(bus, auth, presence, { + handleOf: (login) => + db.prepare('SELECT handle FROM members WHERE login = ? COLLATE NOCASE').get(login)?.handle as + string | undefined, + countryOf: (request) => (config.dev ? countryOf(request, true) : countryOf(request)), +}); const writeback = createWriteback(db, bus, github); const sync = createSync(db, bus, sessions, writeback); const settings = createSettings(db, bus); const snapshot = createSnapshot(db, bus, settings); +const views = createViews(db, redis, bus, { + exists: (kind, key) => + kind === 'post' + ? snapshot.get().posts.some((post) => post.id === key) + : snapshot.get().projects.some((project) => project.slug === key), + allow: (address) => slidingLimit(redis, `views:${address}`, 30, MINUTE), +}); const jobs = createJobs( db, { @@ -75,7 +90,7 @@ const subscribers = createSubscribers(db, jobs, bus, { testAddress: config.alertsEmail, }); const posts = createPosts(db, bus, jobs, subscribers.queuePost); -const collab = createCollab(db, posts, sessions); +const collab = createCollab(db, posts, sessions, bus); process.on('unhandledRejection', (reason) => capture(reason, { route: 'process' })); process.on('uncaughtExceptionMonitor', (error) => capture(error, { route: 'process' })); @@ -120,6 +135,8 @@ const app = createApp( posts, subscribers, collab, + presence, + views, auth, live, sync, @@ -134,6 +151,7 @@ const server = app.listen(config.port, () => log.info({ port: config.port }, 'li server.on('upgrade', collab.upgrade); subscribers.start(); jobs.start(); +presence.start().catch((error: unknown) => capture(error, { route: 'presence' })); function every(name: string, ms: number, job: () => Promise) { const run = () => job().catch((error: unknown) => capture(error, { route: `job:${name}` })); @@ -164,6 +182,7 @@ process.on('SIGTERM', () => { log.info('shutting down'); timers.forEach(clearTimeout); jobs.stop(); + presence.stop(); collab.destroy(); server.close(); server.closeAllConnections(); diff --git a/site/server/migrations/008-views.sql b/site/server/migrations/008-views.sql new file mode 100644 index 0000000..8561263 --- /dev/null +++ b/site/server/migrations/008-views.sql @@ -0,0 +1,7 @@ +CREATE TABLE views ( + kind TEXT NOT NULL CHECK (kind IN ('post', 'project')), + key TEXT NOT NULL, + day TEXT NOT NULL, + views INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (kind, key, day) +); diff --git a/site/server/presence.test.ts b/site/server/presence.test.ts new file mode 100644 index 0000000..851cfe7 --- /dev/null +++ b/site/server/presence.test.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { createPresence } from './presence.ts'; +import { fakeRedis, recordingBus } from './testing.ts'; + +function setup() { + let now = 1_762_500_000_000; + const redis = fakeRedis(); + const { bus, published } = recordingBus(); + const presence = createPresence(redis, bus, () => now); + return { presence, published, redis, advance: (ms: number) => (now += ms) }; +} + +const last = (published: { topic: string; detail: Record }[], topic: string) => + published.filter((event) => event.topic === topic).at(-1)?.detail; + +describe('presence', () => { + it('counts tabs on the site and members online, publishing only on change', async () => { + const { presence, published } = setup(); + await presence.join('t1', {}); + await presence.join('t2', { handle: 'amrmscll' }); + await presence.join('t3', { handle: 'amrmscll' }); + assert.deepEqual(last(published, 'presence'), { here: 3, online: ['amrmscll'] }); + assert.equal(published.filter((event) => event.topic === 'presence').length, 3); + await presence.leave('t2'); + assert.deepEqual(last(published, 'presence'), { here: 2, online: ['amrmscll'] }); + await presence.leave('t3'); + assert.deepEqual(presence.current(), { here: 1, online: [] }); + await presence.touch('t1'); + assert.equal(published.filter((event) => event.topic === 'presence').length, 5); + }); + + it('counts readers per key and forgets a tab that stops touching', async () => { + const { presence, published, advance } = setup(); + await presence.join('t1', { reading: 'post-1' }); + await presence.join('t2', { reading: 'post-1' }); + assert.deepEqual(last(published, 'reading'), { key: 'post-1', reading: 2 }); + assert.equal(presence.reading('post-1'), 2); + advance(60_000); + await presence.touch('t1'); + advance(45_000); + await presence.sweep(); + assert.deepEqual(last(published, 'reading'), { key: 'post-1', reading: 1 }); + assert.deepEqual(last(published, 'presence'), { here: 1, online: [] }); + }); + + it('lights a country for ten minutes after its last tab', async () => { + const { presence, published, advance } = setup(); + await presence.join('t1', { country: 'DE' }); + await presence.join('t2', { country: 'DE' }); + await presence.join('t3', { country: 'EG' }); + assert.deepEqual(presence.countries(), [ + { code: 'DE', count: 2 }, + { code: 'EG', count: 1 }, + ]); + assert.equal(published.filter((event) => event.topic === 'map').at(-1)?.key, 'eg'); + await presence.leave('t1'); + await presence.leave('t2'); + await presence.leave('t3'); + advance(300_000); + await presence.sweep(); + assert.equal(presence.countries().length, 2); + advance(301_000); + await presence.sweep(); + assert.deepEqual(presence.countries(), []); + assert.deepEqual(last(published, 'map'), { country: 'EG', count: 0 }); + }); + + it('keeps nothing but counters in Redis', async () => { + const { presence, redis } = setup(); + await presence.join('t1', { reading: 'post-1', handle: 'amrmscll', country: 'DE' }); + assert.deepEqual(redis.keys().sort(), [ + 'map:countries', + 'presence:online', + 'presence:reading:post-1', + 'presence:tabs', + ]); + }); +}); diff --git a/site/server/presence.ts b/site/server/presence.ts new file mode 100644 index 0000000..e8cda1d --- /dev/null +++ b/site/server/presence.ts @@ -0,0 +1,177 @@ +import type { Country, Presence as Site } from '../app/types.ts'; +import type { Bus } from './bus.ts'; +import type { Redis } from './redis.ts'; + +const WINDOW_MS = 90_000; +const MAP_MS = 600_000; +const SWEEP_MS = 30_000; + +export interface Tab { + reading?: string; + handle?: string; + country?: string; +} + +export function createPresence(redis: Redis, bus: Bus, now = () => Date.now()) { + const tabs = new Map(); + const last = { + site: { here: 0, online: [] as string[] } satisfies Site, + reading: new Map(), + map: new Map(), + }; + let sweeper: ReturnType | undefined; + + const readingKey = (key: string) => `presence:reading:${key}`; + + async function members(key: string, windowMs: number) { + const at = now(); + await redis.zremrangebyscore(key, '-inf', `(${at - windowMs}`); + await redis.expire(key, Math.ceil((windowMs * 2) / 1000)); + return redis.zrange(key, '0', '-1'); + } + + async function add(key: string, member: string) { + await redis.zadd(key, String(now()), member); + } + + async function publish(topic: string, key: string | undefined, detail: Record) { + await bus.publish({ + topic, + key, + kind: topic, + public: true, + summary: `${topic} changed`, + detail, + at: new Date(now()).toISOString(), + }); + } + + async function site() { + const here = (await members('presence:tabs', WINDOW_MS)).length; + const online = [ + ...new Set((await members('presence:online', WINDOW_MS)).map((entry) => entry.split('|')[0])), + ].sort(); + if (here === last.site.here && online.join(',') === last.site.online.join(',')) { + return; + } + last.site = { here, online }; + await publish('presence', undefined, { here, online }); + } + + async function reading(key: string) { + const count = (await members(readingKey(key), WINDOW_MS)).length; + if ((last.reading.get(key) ?? 0) === count) { + return; + } + if (count === 0) { + last.reading.delete(key); + } else { + last.reading.set(key, count); + } + await publish('reading', key, { key, reading: count }); + } + + async function map() { + const counts = new Map(); + for (const entry of await members('map:countries', MAP_MS)) { + const code = entry.split('|')[0]; + counts.set(code, (counts.get(code) ?? 0) + 1); + } + for (const code of new Set([...counts.keys(), ...last.map.keys()])) { + const count = counts.get(code) ?? 0; + if ((last.map.get(code) ?? 0) === count) { + continue; + } + if (count === 0) { + last.map.delete(code); + } else { + last.map.set(code, count); + } + await publish('map', code.toLowerCase(), { country: code, count }); + } + } + + async function write(tab: string, attributes: Tab) { + await add('presence:tabs', tab); + if (attributes.handle) { + await add('presence:online', `${attributes.handle}|${tab}`); + } + if (attributes.reading) { + await add(readingKey(attributes.reading), tab); + } + if (attributes.country) { + await add('map:countries', `${attributes.country}|${tab}`); + } + } + + async function sweep() { + await site(); + for (const key of new Set([ + ...last.reading.keys(), + ...[...tabs.values()].flatMap((tab) => tab.reading ?? []), + ])) { + await reading(key); + } + await map(); + } + + return { + async join(tab: string, attributes: Tab) { + tabs.set(tab, attributes); + await write(tab, attributes); + await site(); + if (attributes.reading) { + await reading(attributes.reading); + } + if (attributes.country) { + await map(); + } + }, + + async touch(tab: string) { + const attributes = tabs.get(tab); + if (attributes) { + await write(tab, attributes); + } + }, + + async leave(tab: string) { + const attributes = tabs.get(tab); + if (!attributes) { + return; + } + tabs.delete(tab); + await redis.zrem('presence:tabs', tab); + if (attributes.handle) { + await redis.zrem('presence:online', `${attributes.handle}|${tab}`); + } + if (attributes.reading) { + await redis.zrem(readingKey(attributes.reading), tab); + } + await site(); + if (attributes.reading) { + await reading(attributes.reading); + } + }, + + sweep, + + start() { + sweeper = setInterval(() => void sweep().catch(() => undefined), SWEEP_MS); + return sweep(); + }, + + stop() { + clearInterval(sweeper); + }, + + current: (): Site => last.site, + reading: (key: string) => last.reading.get(key) ?? 0, + countries: (): Country[] => + [...last.map] + .map(([code, count]) => ({ code, count })) + .sort((a, b) => b.count - a.count || a.code.localeCompare(b.code)), + }; +} + +export type Presence = ReturnType; diff --git a/site/server/sync.ts b/site/server/sync.ts index 8331da9..74c092f 100644 --- a/site/server/sync.ts +++ b/site/server/sync.ts @@ -27,7 +27,16 @@ export function changedFiles(push: Push) { export function createSync(db: Database, bus: Bus, sessions: Sessions, writeback: Writeback) { async function members() { for (const login of await syncMembers(db, bus)) { - await sessions.revoke(login); + const removed = await sessions.revoke(login); + await bus.publish({ + topic: 'sessions', + key: login.toLowerCase(), + kind: 'revoked', + public: false, + summary: `${login} left the organisation`, + detail: { removed }, + at: new Date().toISOString(), + }); } } diff --git a/site/server/testing.ts b/site/server/testing.ts index d9ea3aa..4cb2915 100644 --- a/site/server/testing.ts +++ b/site/server/testing.ts @@ -1,10 +1,64 @@ import type { Bus, HubEvent } from './bus.ts'; import { openDatabase } from './db.ts'; +import type { Redis } from './redis.ts'; export function memoryDatabase() { return openDatabase(':memory:'); } +export function fakeRedis() { + const sorted = new Map>(); + const sets = new Map>(); + const strings = new Map(); + const zset = (key: string) => sorted.get(key) ?? sorted.set(key, new Map()).get(key)!; + const bound = (value: string) => + value === '-inf' ? -Infinity : value === '+inf' ? Infinity : Number(value.replace('(', '')); + const fake = { + async zadd(key: string, score: number, member: string) { + zset(key).set(member, score); + return 1; + }, + async zrem(key: string, ...members: string[]) { + return members.filter((member) => zset(key).delete(member)).length; + }, + async zremrangebyscore(key: string, min: string, max: string) { + const exclusiveMax = max.startsWith('('); + let removed = 0; + for (const [member, score] of zset(key)) { + if (score >= bound(min) && (exclusiveMax ? score < bound(max) : score <= bound(max))) { + zset(key).delete(member); + removed += 1; + } + } + return removed; + }, + async zrange(key: string) { + return [...zset(key)].sort((a, b) => a[1] - b[1]).map(([member]) => member); + }, + async expire() { + return 1; + }, + async pfadd(key: string, member: string) { + const set = sets.get(key) ?? sets.set(key, new Set()).get(key)!; + const fresh = !set.has(member); + set.add(member); + return fresh ? 1 : 0; + }, + async set(key: string, value: string, ...flags: unknown[]) { + if (flags.includes('NX') && strings.has(key)) { + return null; + } + strings.set(key, value); + return 'OK'; + }, + async get(key: string) { + return strings.get(key) ?? null; + }, + keys: () => [...sorted.keys(), ...sets.keys(), ...strings.keys()], + }; + return fake as unknown as Redis & { keys(): string[] }; +} + export function recordingBus() { const recorded: HubEvent[] = []; const published: HubEvent[] = []; diff --git a/site/server/views.test.ts b/site/server/views.test.ts new file mode 100644 index 0000000..50bf1a1 --- /dev/null +++ b/site/server/views.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { fakeRedis, memoryDatabase, recordingBus } from './testing.ts'; +import { createViews, hashAddress, visitorId } from './views.ts'; + +const CHROME = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/142.0.0.0 Safari/537.36'; + +function setup(allowed = () => true) { + let now = 1_762_500_000_000; + const db = memoryDatabase(); + const redis = fakeRedis(); + const { bus, published } = recordingBus(); + const views = createViews(db, redis, bus, { + exists: (kind, key) => key === (kind === 'post' ? 'abc123' : 'strata'), + allow: async () => allowed(), + now: () => now, + }); + const view = (overrides: Partial[0]> = {}) => + views.count({ + kind: 'post', + key: 'abc123', + address: '203.0.113.7', + userAgent: CHROME, + ...overrides, + }); + return { db, redis, views, published, view, advance: (ms: number) => (now += ms) }; +} + +describe('views', () => { + it('counts a visitor once a day per item, so a thousand refreshes add one', async () => { + const { views, view, published } = setup(); + assert.equal(await view(), 'counted'); + for (let refresh = 0; refresh < 999; refresh += 1) { + assert.equal(await view(), 'seen'); + } + assert.equal(views.total('post', 'abc123'), 1); + assert.equal(published.filter((event) => event.topic === 'views').length, 1); + assert.deepEqual(published[0].detail, { kind: 'post', key: 'abc123', total: 1 }); + }); + + it('counts another visitor, another day and another item separately', async () => { + const { views, view, advance, db } = setup(); + await view(); + assert.equal(await view({ address: '198.51.100.9' }), 'counted'); + assert.equal(await view({ userAgent: `${CHROME} Edg/142` }), 'counted'); + assert.equal(await view({ kind: 'project', key: 'strata' }), 'counted'); + advance(86_400_000); + assert.equal(await view(), 'counted'); + assert.equal(views.total('post', 'abc123'), 4); + assert.equal(views.total('project', 'strata'), 1); + assert.equal(db.prepare('SELECT count(*) AS n FROM views').get()!.n, 3); + assert.deepEqual(views.totals(), { 'post:abc123': 4, 'project:strata': 1 }); + }); + + it('refuses unknown items, bots and anyone over the per-minute limit', async () => { + const { view } = setup(); + assert.equal(await view({ key: 'missing' }), 'unknown'); + assert.equal(await view({ kind: 'page' }), 'unknown'); + assert.equal(await view({ userAgent: 'Mozilla/5.0 (compatible; Googlebot/2.1)' }), 'bot'); + assert.equal(await view({ userAgent: 'curl/8.4.0' }), 'bot'); + assert.equal(await view({ userAgent: '' }), 'bot'); + assert.equal(await setup(() => false).view(), 'limited'); + }); + + it('stores hashes only: no address, agent or visitor id in Redis', async () => { + const { view, redis } = setup(); + await view(); + const keys = redis.keys(); + assert.deepEqual(keys.sort(), ['views:post:abc123:2025-11-07', 'views:salt:2025-11-07']); + assert.ok(!JSON.stringify(keys).includes('203.0.113.7')); + assert.notEqual(visitorId('a', '203.0.113.7', CHROME), visitorId('b', '203.0.113.7', CHROME)); + assert.equal(hashAddress('203.0.113.7').length, 22); + }); +}); diff --git a/site/server/views.ts b/site/server/views.ts new file mode 100644 index 0000000..a3195b8 --- /dev/null +++ b/site/server/views.ts @@ -0,0 +1,100 @@ +import { createHash, randomBytes } from 'node:crypto'; +import type { Bus } from './bus.ts'; +import type { Database } from './db.ts'; +import type { Redis } from './redis.ts'; + +export type ViewKind = 'post' | 'project'; + +const KINDS = new Set(['post', 'project']); +const KEY = /^[a-z0-9-]{1,80}$/; +const BOT = + /bot|crawl|spider|slurp|headless|lighthouse|pagespeed|preview|fetch|curl|wget|python|java\/|go-http|okhttp|monitor|scan/i; +const TWO_DAYS = 2 * 86_400; + +export function isViewKind(value: string): value is ViewKind { + return KINDS.has(value); +} + +export function hashAddress(address: string) { + return createHash('sha256').update(address).digest('base64url').slice(0, 22); +} + +export function visitorId(salt: string, address: string, userAgent: string) { + return createHash('sha256') + .update(`${salt}\n${address}\n${userAgent}`) + .digest('base64url') + .slice(0, 22); +} + +export function createViews( + db: Database, + redis: Redis, + bus: Bus, + deps: { + exists: (kind: ViewKind, key: string) => boolean; + allow: (addressHash: string) => Promise; + now?: () => number; + }, +) { + const now = deps.now ?? (() => Date.now()); + const totals = new Map( + db + .prepare('SELECT kind, key, sum(views) AS total FROM views GROUP BY kind, key') + .all() + .map((row) => [`${row.kind as string}:${row.key as string}`, row.total as number]), + ); + const bump = db.prepare( + `INSERT INTO views (kind, key, day, views) VALUES (?, ?, ?, 1) + ON CONFLICT (kind, key, day) DO UPDATE SET views = views + 1`, + ); + + async function salt(day: string) { + const key = `views:salt:${day}`; + await redis.set(key, randomBytes(16).toString('hex'), 'EX', TWO_DAYS, 'NX'); + return (await redis.get(key)) ?? ''; + } + + return { + totals: () => Object.fromEntries(totals), + + total: (kind: ViewKind, key: string) => totals.get(`${kind}:${key}`) ?? 0, + + async count(view: { kind: string; key: string; address: string; userAgent: string }) { + const { kind, key } = view; + if (!isViewKind(kind) || !KEY.test(key) || !deps.exists(kind, key)) { + return 'unknown'; + } + if (!view.userAgent || BOT.test(view.userAgent)) { + return 'bot'; + } + if (!(await deps.allow(hashAddress(view.address)))) { + return 'limited'; + } + const day = new Date(now()).toISOString().slice(0, 10); + const bucket = `views:${kind}:${key}:${day}`; + const fresh = await redis.pfadd( + bucket, + visitorId(await salt(day), view.address, view.userAgent), + ); + await redis.expire(bucket, TWO_DAYS); + if (!fresh) { + return 'seen'; + } + bump.run(kind, key, day); + const total = (totals.get(`${kind}:${key}`) ?? 0) + 1; + totals.set(`${kind}:${key}`, total); + await bus.publish({ + topic: 'views', + key, + kind: 'views', + public: true, + summary: `${kind} ${key} viewed`, + detail: { kind, key, total }, + at: new Date(now()).toISOString(), + }); + return 'counted'; + }, + }; +} + +export type Views = ReturnType; From 179b1b53a92910d6ac2705ac9e911580d023fd1a Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:10:12 +0200 Subject: [PATCH 3/4] feat(live): the live pulse, online dots, view counts and the visitor map on /status --- site/app/components/Shell.tsx | 2 + site/app/components/kit/basics.tsx | 10 +- site/app/components/kit/live.tsx | 72 ++++++++- site/app/components/kit/map-data.ts | 220 +++++++++++++++++++++++++++ site/app/components/kit/map.tsx | 82 ++++++++++ site/app/components/kit/members.tsx | 24 ++- site/app/components/kit/panels.tsx | 4 +- site/app/components/kit/posts.tsx | 2 + site/app/components/kit/projects.tsx | 16 +- site/app/styles/graphite.css | 116 +++++++++++++- site/app/styles/pages/live.css | 94 ++++++++++++ 11 files changed, 626 insertions(+), 16 deletions(-) create mode 100644 site/app/components/kit/map-data.ts create mode 100644 site/app/components/kit/map.tsx diff --git a/site/app/components/Shell.tsx b/site/app/components/Shell.tsx index eeb40ce..62656e0 100644 --- a/site/app/components/Shell.tsx +++ b/site/app/components/Shell.tsx @@ -6,6 +6,7 @@ import { MotionContext } from '../motion/motion'; import { defaults, visible, type NAV } from '../settings'; import AccountMenu from './AccountMenu'; import { Editable, useEditing } from './kit/Editable'; +import { LivePulse } from './kit/live'; import { openSearch, Palette } from './kit/Palette'; import { SubscribeForm, SubscribeLink } from './kit/subscribe'; @@ -70,6 +71,7 @@ export default function Shell({ children }: { children: ReactNode }) { search / + {root && } diff --git a/site/app/components/kit/basics.tsx b/site/app/components/kit/basics.tsx index 798c10d..fbc4ee6 100644 --- a/site/app/components/kit/basics.tsx +++ b/site/app/components/kit/basics.tsx @@ -10,12 +10,18 @@ export const accentStyle = (accent: string | undefined) => interface AvatarProps { member: { name: string; avatarUrl: string; accent: string }; size?: 'xs' | 'sm' | 'md' | 'lg'; + online?: boolean; } -export function Avatar({ member, size }: AvatarProps) { +export function Avatar({ member, size, online = false }: AvatarProps) { const pixels = size ? { xs: 20, sm: 32, md: 40, lg: 64 }[size] : 48; return ( -