Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions apps/api/src/routes/internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { promisify } from 'node:util';
import type { FastifyInstance } from 'fastify';

import { errorResponse, ok } from '../lib/response.js';
import { reloadInMemoryStateAndFts } from '../store/memory/reload.js';
import { reloadInMemoryStateAndFts, type ReloadMode } from '../store/memory/reload.js';
import type { ReconcileOutcome, ReconcileResult } from '../store/reconcile.js';

const exec = promisify(execFile);
Expand Down Expand Up @@ -197,9 +197,12 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {

// ---- Reconcile + rebuild under the repository write lock ----
let result: ReconcileResult;
let reloadMode: ReloadMode | undefined;
try {
result = await fastify.reconcileDataRepo({
onMoved: () => reloadInMemoryStateAndFts(fastify),
onMoved: async (moved) => {
reloadMode = await reloadInMemoryStateAndFts(fastify, { fromCommit: moved.oldCommit });
},
});
} catch (err) {
// A throw from the rebuild leaves the in-memory state + FTS index
Expand Down Expand Up @@ -242,6 +245,7 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {
oldCommit: result.oldCommit,
newCommit: result.newCommit,
conflictBranch: result.conflictBranch,
reloadMode,
durationMs,
},
'hot-reload: in-memory state + FTS rebuilt',
Expand All @@ -251,6 +255,7 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {
ok({
noChanges: false,
rebuilt: true,
reloadMode,
outcome: result.outcome,
oldCommit: result.oldCommit,
newCommit: result.newCommit,
Expand Down
170 changes: 170 additions & 0 deletions apps/api/src/store/memory/incremental.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
/**
* Incremental hot reload — specs/behaviors/storage.md → "Hot reload".
*
* Instead of re-reading every sheet, diff each in-memory sheet between the
* commit the live state reflects and the new HEAD (gitsheets
* `Sheet.diffFrom`), then unindex each changed record's old version and
* index its new one. `unindex*` is the exact inverse of `index*`, so the
* result equals a full load of the new tree.
*/
import { RECORD_PATH_KEY, RECORD_SHEET_KEY, type DiffChange } from 'gitsheets';

import type { FtsEngine } from '../fts.js';
import type { PublicStore } from '../public.js';
import {
indexBlogPost,
indexHelpWantedInterest,
indexHelpWantedRole,
indexMembership,
indexPerson,
indexPersonEvaluation,
indexProject,
indexProjectBuzz,
indexProjectUpdate,
indexSlugHistory,
indexTag,
indexTagAssignment,
unindexBlogPost,
unindexHelpWantedInterest,
unindexHelpWantedRole,
unindexMembership,
unindexPerson,
unindexPersonEvaluation,
unindexProject,
unindexProjectBuzz,
unindexProjectUpdate,
unindexSlugHistory,
unindexTag,
unindexTagAssignment,
type InMemoryState,
} from './state.js';

/** More changed records than this and a full rebuild is cheaper and safer. */
export const INCREMENTAL_RECORD_LIMIT = 5_000;

type Sheets = PublicStore;
type RecordOf<K extends keyof Sheets> = Sheets[K] extends { queryAll(): Promise<(infer T)[]> }
? T
: never;

interface SheetHandler<K extends keyof Sheets> {
readonly sheet: K;
readonly unindex: (state: InMemoryState, record: RecordOf<K>) => void;
readonly index: (state: InMemoryState, record: RecordOf<K>) => void;
readonly ftsRemove?: (fts: FtsEngine, record: RecordOf<K>) => void;
readonly ftsAdd?: (fts: FtsEngine, record: RecordOf<K>) => void;
}

function handler<K extends keyof Sheets>(h: SheetHandler<K>): SheetHandler<K> {
return h;
}

/** Every sheet the loader reads into InMemoryState, in loader order. */
const HANDLERS = [
handler({
sheet: 'projects',
unindex: unindexProject,
index: indexProject,
ftsRemove: (fts, p) => fts.removeProject(p.slug),
ftsAdd: (fts, p) => {
if (!p.deletedAt) fts.upsertProject(p.slug, p.title, p.summary ?? '', p.overview ?? '');
},
}),
handler({
sheet: 'people',
unindex: unindexPerson,
index: indexPerson,
ftsRemove: (fts, p) => fts.removePerson(p.slug),
ftsAdd: (fts, p) => {
if (!p.deletedAt) fts.upsertPerson(p.slug, p.fullName, p.bio ?? '');
},
}),
handler({ sheet: 'tags', unindex: unindexTag, index: indexTag }),
handler({ sheet: 'tag-assignments', unindex: unindexTagAssignment, index: indexTagAssignment }),
handler({ sheet: 'project-memberships', unindex: unindexMembership, index: indexMembership }),
handler({ sheet: 'project-updates', unindex: unindexProjectUpdate, index: indexProjectUpdate }),
handler({ sheet: 'project-buzz', unindex: unindexProjectBuzz, index: indexProjectBuzz }),
handler({ sheet: 'blog-posts', unindex: unindexBlogPost, index: indexBlogPost }),
handler({
sheet: 'help-wanted-roles',
unindex: unindexHelpWantedRole,
index: indexHelpWantedRole,
ftsRemove: (fts, r) => fts.removeHelpWanted(r.id),
ftsAdd: (fts, r) => fts.upsertHelpWanted(r.id, r.title, r.description),
}),
handler({
sheet: 'help-wanted-interest',
unindex: unindexHelpWantedInterest,
index: indexHelpWantedInterest,
}),
handler({
sheet: 'slug-history',
unindex: unindexSlugHistory,
index: (state, r) => indexSlugHistory(state, r),
}),
handler({
sheet: 'person-evaluations',
unindex: unindexPersonEvaluation,
index: indexPersonEvaluation,
}),
] as const;

interface CollectedChanges {
readonly handler: SheetHandler<keyof Sheets>;
readonly changes: readonly DiffChange[];
}

/**
* Collect every changed record between `fromCommit` and each sheet's current
* read snapshot. Returns null when more than {@link INCREMENTAL_RECORD_LIMIT}
* records changed, meaning the caller should do a full rebuild.
*/
export async function collectChanges(
store: PublicStore,
fromCommit: string,
): Promise<CollectedChanges[] | null> {
const collected: CollectedChanges[] = [];
let total = 0;
for (const h of HANDLERS) {
const changes: DiffChange[] = [];
for await (const change of store[h.sheet].diffFrom(fromCommit, { records: true })) {
changes.push(change);
if (++total > INCREMENTAL_RECORD_LIMIT) return null;
}
if (changes.length > 0) {
collected.push({ handler: h as unknown as SheetHandler<keyof Sheets>, changes });
}
}
return collected;
}

/**
* Apply collected changes to the live state and FTS index. Synchronous, so
* no request handler observes a half-applied diff.
*/
export function applyChanges(
state: InMemoryState,
fts: FtsEngine,
collected: readonly CollectedChanges[],
): number {
let applied = 0;
for (const { handler: h, changes } of collected) {
for (const change of changes) {
const src = change.src as RecordOf<keyof Sheets> | undefined;
const dst = change.dst as RecordOf<keyof Sheets> | undefined;
if (src) {
h.unindex(state, src);
h.ftsRemove?.(fts, src);
}
if (dst) {
// Annotate like `queryAll()` does: gitsheets locates a record's
// current path through these on a later upsert/delete.
Object.assign(dst, { [RECORD_SHEET_KEY]: h.sheet, [RECORD_PATH_KEY]: change.path });
h.index(state, dst);
h.ftsAdd?.(fts, dst);
}
applied++;
}
}
return applied;
}
52 changes: 48 additions & 4 deletions apps/api/src/store/memory/reload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,36 @@
* Used by `apps/api/src/routes/internal.ts` (the `POST
* /api/_internal/reload-data` webhook).
*/
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';

import type { FastifyInstance } from 'fastify';
import { invalidateFacets } from './facets.js';
import { applyChanges, collectChanges } from './incremental.js';
import { loadInMemoryState } from './loader.js';
import type { InMemoryState } from './state.js';

const exec = promisify(execFile);

export interface ReloadOptions {
/**
* The commit the live state currently reflects. When given, the reload
* applies only the records that changed since, unless a sheet config
* changed or the diff is too large.
*/
readonly fromCommit?: string;
}

export type ReloadMode = 'incremental' | 'full';

/** Whether `.gitsheets/` (the sheet configs) differs between two commits. */
async function sheetConfigsChanged(repoPath: string, from: string, to: string): Promise<boolean> {
const tree = async (commit: string): Promise<string> =>
(await exec('git', ['rev-parse', `${commit}:.gitsheets`], { cwd: repoPath })).stdout.trim();
const [a, b] = await Promise.all([tree(from), tree(to)]);
return a !== b;
}

/**
* Replace the contents of `target`'s Maps with `source`'s Maps. The
* `target` object identity is preserved so services that captured a
Expand All @@ -42,16 +67,34 @@ function replaceMapContents<K, V>(target: Map<K, V>, source: Map<K, V>): void {
}

/**
* Build a fresh `InMemoryState` from the public store and swap it into
* the running Fastify instance, then reload the revocation set and the FTS
* index from the same snapshot and invalidate the facet cache.
* Bring the live in-memory state, revocation set and FTS index up to the
* public store's current snapshot, and invalidate the facet cache.
*
* With `fromCommit`, applies only the records that changed since (see
* `incremental.ts`); otherwise, or when configs changed or the diff is
* large, rebuilds everything. Returns which path ran.
*
* The caller must already have refreshed the public store to the new HEAD
* (`fastify.reconcileDataRepo` does, under the repository write lock).
*/
export async function reloadInMemoryStateAndFts(
fastify: FastifyInstance,
): Promise<void> {
opts: ReloadOptions = {},
): Promise<ReloadMode> {
const { fromCommit } = opts;
if (fromCommit) {
const repoPath = fastify.config.CFP_DATA_REPO_PATH;
if (!(await sheetConfigsChanged(repoPath, fromCommit, 'HEAD'))) {
const changes = await collectChanges(fastify.store.public, fromCommit);
if (changes) {
applyChanges(fastify.inMemoryState, fastify.fts, changes);
await fastify.revocations.loadFromSheet(fastify.store.public);
invalidateFacets();
return 'incremental';
}
}
}

// Step 1: build the fresh in-memory state. If this throws (validator
// failure on a new record, etc.), the live state is untouched and the
// caller surfaces the error.
Expand All @@ -74,6 +117,7 @@ export async function reloadInMemoryStateAndFts(

// Step 5: drop the cached facet roll-ups so the next request recomputes.
invalidateFacets();
return 'full';
}

/**
Expand Down
97 changes: 97 additions & 0 deletions apps/api/src/store/memory/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,100 @@ export function indexPersonEvaluation(state: InMemoryState, record: PersonEvalua
}
set.add(key);
}

// ---------------------------------------------------------------------------
// Unindexing — the exact inverse of each index* above, so applying a commit
// diff (unindex the old version, index the new one) leaves the state equal to
// a full load of the new tree. Empty secondary Sets are deleted, since a full
// load never creates them. Used by the incremental hot reload.
// ---------------------------------------------------------------------------

function removeFromSet<K>(map: Map<K, Set<string>>, key: K, id: string): void {
const set = map.get(key);
if (!set) return;
set.delete(id);
if (set.size === 0) map.delete(key);
}

/** Delete `key` from `map` only if it still points at `id`. */
function deleteIfOwned<K>(map: Map<K, string>, key: K, id: string): void {
if (map.get(key) === id) map.delete(key);
}

export function unindexSlugHistory(state: InMemoryState, record: SlugHistory): void {
const key = slugHistoryKey(record.entityType, record.oldSlug);
if (state.slugHistory.get(key)?.newSlug === record.newSlug) state.slugHistory.delete(key);
}

export function unindexProject(state: InMemoryState, project: Project): void {
state.projects.delete(project.id);
state.projectSlugById.delete(project.id);
deleteIfOwned(state.projectIdBySlug, project.slug, project.id);
if (typeof project.legacyId === 'number') {
deleteIfOwned(state.projectIdByLegacyId, project.legacyId, project.id);
}
}

export function unindexPerson(state: InMemoryState, person: Person): void {
state.people.delete(person.id);
state.personSlugById.delete(person.id);
deleteIfOwned(state.personIdBySlug, person.slug, person.id);
}

export function unindexTag(state: InMemoryState, tag: Tag): void {
state.tags.delete(tag.id);
deleteIfOwned(state.tagIdByHandle, `${tag.namespace}.${tag.slug}`, tag.id);
}

export function unindexTagAssignment(state: InMemoryState, ta: TagAssignment): void {
state.tagAssignments.delete(ta.id);
removeFromSet(state.tagAssignmentsByTaggable, ta.taggableId, ta.id);
removeFromSet(state.tagAssignmentsByTag, ta.tagId, ta.id);
}

export function unindexMembership(state: InMemoryState, m: ProjectMembership): void {
state.projectMemberships.delete(m.id);
removeFromSet(state.membershipsByProject, m.projectId, m.id);
removeFromSet(state.membershipsByPerson, m.personId, m.id);
}

export function unindexProjectUpdate(state: InMemoryState, update: ProjectUpdate): void {
state.projectUpdates.delete(update.id);
removeFromSet(state.updatesByProject, update.projectId, update.id);
deleteIfOwned(state.updateByProjectAndNumber, `${update.projectId}:${update.number}`, update.id);
}

export function unindexProjectBuzz(state: InMemoryState, buzz: ProjectBuzz): void {
state.projectBuzz.delete(buzz.id);
removeFromSet(state.buzzByProject, buzz.projectId, buzz.id);
deleteIfOwned(state.buzzByProjectAndSlug, `${buzz.projectId}:${buzz.slug}`, buzz.id);
deleteIfOwned(state.buzzIdBySlug, buzz.slug, buzz.id);
}

export function unindexBlogPost(state: InMemoryState, post: BlogPost): void {
state.blogPosts.delete(post.id);
deleteIfOwned(state.blogPostIdBySlug, post.slug, post.id);
if (typeof post.legacyId === 'number') {
deleteIfOwned(state.blogPostIdByLegacyId, post.legacyId, post.id);
}
}

export function unindexHelpWantedRole(state: InMemoryState, role: HelpWantedRole): void {
state.helpWantedRoles.delete(role.id);
removeFromSet(state.helpWantedByProject, role.projectId, role.id);
}

export function unindexHelpWantedInterest(
state: InMemoryState,
expr: HelpWantedInterestExpression,
): void {
state.helpWantedInterest.delete(expr.id);
removeFromSet(state.interestByRole, expr.roleId, expr.id);
deleteIfOwned(state.interestByRoleAndPerson, `${expr.roleId}:${expr.personId}`, expr.id);
}

export function unindexPersonEvaluation(state: InMemoryState, record: PersonEvaluation): void {
const key = `${record.personSlug}/${record.evaluator}`;
state.personEvaluations.delete(key);
removeFromSet(state.evaluationsByPerson, record.personSlug, key);
}
Loading
Loading