diff --git a/deploy/nginx/common/routes-core.conf b/deploy/nginx/common/routes-core.conf index a7f8974b..d7e28596 100644 --- a/deploy/nginx/common/routes-core.conf +++ b/deploy/nginx/common/routes-core.conf @@ -74,6 +74,17 @@ location = /ContentService/data/species/all { proxy_pass http://content_node; } +# An object with its neighbours, their species, and their regulations, +# references and markers one step further: graph-core's +# findOldEnhancedObjectById, written by the domain serialiser. The entity +# popup, the website's content pages and the IDG panel read it. Not +# /enhanced/v2/{id}, which Java still serves. Byte-identical to Java over +# 6,000 objects of every class. +location ~ ^/ContentService/data/query/enhanced/[^/]+$ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://content_node; +} + # An entity or an event and everything below it (graph-core's get...InDepth), # which the Pathway Browser's object tree reads with JSOG in the # nested-aggregated view. Byte-identical to Java over 1,229 requests: 529 diff --git a/projects/website-angular/src/scripts/proxy-order.spec.ts b/projects/website-angular/src/scripts/proxy-order.spec.ts index c480485a..20d71d2d 100644 --- a/projects/website-angular/src/scripts/proxy-order.spec.ts +++ b/projects/website-angular/src/scripts/proxy-order.spec.ts @@ -70,6 +70,18 @@ describe('the dev server proxy', () => { }); } + for (const url of [ + '/ContentService/data/query/enhanced/R-HSA-70486', + '/ContentService/data/query/enhanced/70486?includeRef=true', + // Java's /query/enhanced/{id} with the id `normalPathway`, not + // /query/{id}/normalPathway with the id `enhanced`. + '/ContentService/data/query/enhanced/normalPathway', + ]) { + it(`sends ${url} to the content service`, async () => { + expect(await targetOf(url)).toBe(contentNode); + }); + } + for (const url of [ '/ContentService/data/entity/R-HSA-109796/in-depth', '/ContentService/data/entity/109796/in-depth?includeRef=true&view=nested-aggregated', @@ -171,17 +183,17 @@ describe('the dev server proxy', () => { '/ContentService/data/query/70486/displayName', '/ContentService/data/query/70486/stIdVersion', '/ContentService/data/query/70486/normalPathways', - // Java's more specific /query/abridged/{id} and /query/enhanced/{id} take - // these, reading the attribute's name as the id. + // Java's more specific /query/abridged/{id} takes this, reading the + // attribute's name as the id. (/query/enhanced/{id} does the same, and + // is node's now: see above.) '/ContentService/data/query/abridged/stId', - '/ContentService/data/query/enhanced/normalPathway', // Java matches attribute names in any case; only the site's spelling is routed. '/ContentService/data/query/70486/NORMALPATHWAY', // Spring also matches a trailing slash; only the plain object is routed. '/ContentService/data/query/70486/', // Java's POST /query/ids (a batch of objects); node serves no POST. '/ContentService/data/query/ids', - '/ContentService/data/query/enhanced/70486', + '/ContentService/data/query/enhanced/v2/70486', '/ContentService/data/query/abridged/70486', '/ContentService/data/person/68285/authoredPathwaysAndMore', // Only the ancestors of an event are ported. diff --git a/projects/website-angular/src/scripts/serve-prod.spec.ts b/projects/website-angular/src/scripts/serve-prod.spec.ts index 43076d5f..a87bf6e6 100644 --- a/projects/website-angular/src/scripts/serve-prod.spec.ts +++ b/projects/website-angular/src/scripts/serve-prod.spec.ts @@ -315,7 +315,8 @@ describe('serve-prod and the API roots', () => { ['/ContentService/data/person/68285/authoredPathwaysAndMore', 'backend'], ['/ContentService/data/query/R-HSA-70486', 'content-node'], ['/ContentService/data/query/R-HSA-70486?view=nested', 'content-node'], - ['/ContentService/data/query/enhanced/R-HSA-70486', 'backend'], + ['/ContentService/data/query/enhanced/R-HSA-70486', 'content-node'], + ['/ContentService/data/query/enhanced/v2/R-HSA-70486', 'backend'], ['/ContentService/data/query/ids', 'backend'], ]) { it(`sends ${url} to ${to}`, async () => { diff --git a/proxy.conf.js b/proxy.conf.js index d93cf639..ca4e931b 100644 --- a/proxy.conf.js +++ b/proxy.conf.js @@ -102,6 +102,9 @@ module.exports = { }, ]) ), + // An object with its neighbours and some of theirs (the older enhanced + // query). Not /enhanced/v2/, which Java still serves. + '^/ContentService/data/query/enhanced/[^/?]+(\\?|$)': contentNode, // An entity or an event and everything below it, for the object tree. '^/ContentService/data/(entity|event)/[^/?]+/in-depth(\\?|$)': contentNode, // A person, and the publications they authored. Not the other attributes diff --git a/tools/content-node/domain.mjs b/tools/content-node/domain.mjs index 176153f9..7eba5514 100644 --- a/tools/content-node/domain.mjs +++ b/tools/content-node/domain.mjs @@ -103,50 +103,139 @@ const isA = (schemaClass, target) => supertypes(schemaClass).has(target); * one SDN fills fields in (see fill); without it, the nodes' own order. */ export function mapResult({ root, nodes, relationships: returned, order }) { - // A relationship returned twice -- on two paths, in-depth -- is one - // relationship to SDN: kept where it first appears. + // extractRelationships / extractNodes: each relationship and node once, in + // the order of first appearance -- the nodes of list columns first, then + // single-node columns, so the root sits where it first appears in a list + // and otherwise last. const seenRelationships = new Set(); - const relationships = returned.filter( - (relationship) => - relationship.id === undefined || - (!seenRelationships.has(relationship.id) && seenRelationships.add(relationship.id)) - ); - const entities = new Map(); - const rootClass = schemaClassOf(root.labels); - entities.set(root.id, new Entity(root.id, rootClass, root.props)); - const byId = new Map(nodes.filter(Boolean).map((node) => [node.id, node])); - // A related node's class is chosen for the root's field that reaches it. - for (const field of SNAPSHOT().graph[rootClass]?.fields ?? []) { - if (field.length === 2) continue; - const [, type, direction, target] = field; - for (const relationship of relationships) { - if (relationship.type !== type) continue; - const outgoing = direction === 'OUTGOING'; - if ((outgoing ? relationship.start : relationship.end) !== root.id) continue; - const node = byId.get(outgoing ? relationship.end : relationship.start); - if (!node || entities.has(node.id)) continue; - entities.set(node.id, new Entity(node.id, classFor(target, node.labels), node.props)); - } - } - for (const node of byId.values()) - if (!entities.has(node.id)) - entities.set(node.id, new Entity(node.id, schemaClassOf(node.labels), node.props)); - // The order SDN visits related nodes in: first appearance in the result. + const relationships = returned + // One without an id (a test's) is its own relationship, under a key of its own. + .map((relationship, at) => + relationship.id === undefined ? { ...relationship, id: `#${at}` } : relationship + ) + .filter( + (relationship) => + !seenRelationships.has(relationship.id) && seenRelationships.add(relationship.id) + ); + const listed = nodes.filter(Boolean); + const byId = new Map(); + for (const node of [...listed, root]) if (!byId.has(node.id)) byId.set(node.id, node); const position = new Map(); - for (const id of [root.id, ...(order ?? nodes.filter(Boolean).map((node) => node.id))]) + for (const id of [...(order ?? listed.map((node) => node.id)), root.id]) if (!position.has(id)) position.set(id, position.size); - // Each relationship under both of its ends, so a field finds its own at once. - const byEnd = new Map(); - const add = (key, relationship) => { - if (!byEnd.has(key)) byEnd.set(key, []); - byEnd.get(key).push(relationship); + const state = { + byId, + position, + relationships, + removed: new Set(), + processed: new Map(), + known: new Map(), + byEnd: new Map(), }; for (const relationship of relationships) { - add(`OUTGOING ${relationship.type} ${relationship.start}`, relationship); - add(`INCOMING ${relationship.type} ${relationship.end}`, relationship); + for (const key of [ + `OUTGOING ${relationship.type} ${relationship.start}`, + `INCOMING ${relationship.type} ${relationship.end}`, + ]) { + if (!state.byEnd.has(key)) state.byEnd.set(key, []); + state.byEnd.get(key).push(relationship); + } + } + return map(state, root, undefined); +} + +/** + * DefaultNeo4jEntityConverter.map and createInstanceOfRelationships: the + * root as an entity, and everything its fields reach. + * + * A node becomes an entity once, its class decided by the field that first + * reaches it (classFor) -- so which field reaches a node first matters, and + * SDN goes depth first: an entity met in a field has its own fields filled + * before that field goes on to the next. Done here with a stack of frames, in + * that order, rather than by recursion, which a deep in-depth answer -- an + * object reached through thousands of others -- would run out of stack on. + * + * Each field: the candidates are the result's nodes that carry the target's + * label, in result order; for each, the field's relationships to it, each + * mapped as it is met. A relationship is counted per direction each time a + * field uses it, and once one has been used twice in the same direction it + * leaves the result for every field after -- SDN's guard against cycles, + * kept for fidelity: on Reactome's model only PhysicalEntity has fields + * sharing a type and direction (three `regulator` INCOMING), a node can match + * at most two, and each owner is mapped once, so it does not fire. Not + * reproduced: SDN gathers a list column's relationships into a HashSet, so two + * parallel edges of one type between the same nodes would come in its hash + * order, not the result's; whether the graph has any is unchecked. Filling node + * by node rather than relationship by relationship is visible when one node + * is reached twice: an entity that is both a reaction's input (order 2) and + * its output (order 1) is the first output, because it was met among the + * inputs; Java writes R-CEL-156804's reaction that way. + */ +function map(state, node, target) { + const frames = []; + const make = (from, to) => { + const schemaClass = to ? classFor(to, from.labels) : schemaClassOf(from.labels); + const entity = new Entity(from.id, schemaClass, from.props); + // Known before its fields are filled, so a field leading back finds it. + state.known.set(from.id, entity); + const graph = SNAPSHOT().graph[schemaClass]; + if (graph) { + for (const field of graph.fields) + if (field.length === 2) + entity.fields[field[0]] = entity.properties[field[1]] ?? graph.defaults?.[field[0]]; + frames.push({ entity, fields: graph.fields.filter((field) => field.length > 2), at: 0 }); + } + return entity; + }; + const root = make(node, target); + while (frames.length) { + const frame = frames[frames.length - 1]; + if (!frame.matching) { + if (frame.at === frame.fields.length) { + frames.pop(); + continue; + } + frame.matching = matchingFor(state, frame.entity, frame.fields[frame.at]); + frame.next = 0; + frame.found = []; + } + const [name, , direction, target, collection, relationClass] = frame.fields[frame.at]; + if (frame.next === frame.matching.length) { + frame.entity.fields[name] = collect(frame.found, collection, relationClass); + frame.matching = undefined; + frame.at += 1; + continue; + } + const relationship = frame.matching[frame.next]; + frame.next += 1; + const key = `${direction} ${relationship.id}`; + const used = state.processed.get(key) ?? 0; + if (used === 2) state.removed.add(relationship.id); + else state.processed.set(key, used + 1); + const otherId = direction === 'OUTGOING' ? relationship.end : relationship.start; + // A new entity pushes its frame: its fields are filled before this one goes on. + const other = + otherId === frame.entity.id + ? frame.entity + : (state.known.get(otherId) ?? make(state.byId.get(otherId), target)); + frame.found.push( + relationClass ? new Relation(relationClass, other, relationship.props) : other + ); } - for (const entity of entities.values()) fill(entity, entities, byEnd, position); - return entities.get(root.id); + return root; +} + +/** One field's relationships, in candidate order (see map). */ +function matchingFor(state, entity, [, type, direction, target]) { + const outgoing = direction === 'OUTGOING'; + const otherEnd = (relationship) => (outgoing ? relationship.end : relationship.start); + return ( + (state.byEnd.get(`${direction} ${type} ${entity.id}`) ?? []) + .filter((relationship) => !state.removed.has(relationship.id)) + .filter((relationship) => state.byId.get(otherEnd(relationship))?.labels.includes(target)) + // Candidate order; stable, so relationships to one node keep theirs. + .sort((a, b) => state.position.get(otherEnd(a)) - state.position.get(otherEnd(b))) + ); } /** @@ -174,42 +263,10 @@ function classFor(target, labels) { return best ?? schemaClassOf(labels); } -/** - * One entity's fields. A relationship field is filled node by node, in the - * order the nodes first appear in the result, and only then relationship by - * relationship -- not in the relationships' own order. The difference shows - * when one node is reached twice: an entity that is both a reaction's input - * (order 2) and its output (order 1) is the *first* output, because it was - * met among the inputs; Java writes R-CEL-156804's reaction that way. - */ -function fill(entity, entities, byEnd, position) { - const graph = SNAPSHOT().graph[entity.schemaClass]; - if (!graph) return; - for (const field of graph.fields) { - if (field.length === 2) { - entity.fields[field[0]] = entity.properties[field[1]] ?? graph.defaults?.[field[0]]; - continue; - } - const [name, type, direction, target, collection, relationClass] = field; - const outgoing = direction === 'OUTGOING'; - const otherEnd = (relationship) => (outgoing ? relationship.end : relationship.start); - // Stable: relationships to the same node keep their own order. - const matching = [...(byEnd.get(`${direction} ${type} ${entity.id}`) ?? [])].sort( - (a, b) => position.get(otherEnd(a)) - position.get(otherEnd(b)) - ); - const found = []; - for (const relationship of matching) { - const other = entities.get(otherEnd(relationship)); - if (!other || !isA(other.schemaClass, target)) continue; - found.push(relationClass ? new Relation(relationClass, other, relationship.props) : other); - } - entity.fields[name] = collect(found, collection, relationClass); - } -} - /** The field's collection as SDN creates it: ArrayList, LinkedHashSet, TreeSet. */ function collect(found, collection, relationClass) { - if (collection === '') return found.length ? found[found.length - 1] : undefined; + // A single value is the first found (createInstanceOfRelationships: value.get(0)). + if (collection === '') return found.length ? found[0] : undefined; if (collection === 'List') return found; // Has.equals compares elements; DatabaseObject.equals, the node. const key = relationClass ? (item) => item.element : (item) => item; @@ -511,6 +568,30 @@ export function inDepthQuery({ label, follow, idType, maxDepth, attributes }) { [r IN relationships | ${rel('r')}] AS relationships`; } +/** + * graph-core's findOldEnhancedObjectById: the object, everything it is + * related to (outgoing only, or both ways), those objects' species, and their + * regulations, physical entities, references and markers one step further. + * + * Java collects every row's nodes, repeats and all, which for a busy object + * is the same nodes many times over. Only the order of first appearance + * matters to the mapping (fill), and collecting each list DISTINCT keeps that + * order: a node first appears where it first appears in its first list. + */ +export function oldEnhancedQuery({ idType, outgoingOnly }) { + const node = (n) => `{id: id(${n}), labels: labels(${n}), props: properties(${n})}`; + const rel = (r) => + `{id: id(${r}), type: type(${r}), start: id(startNode(${r})), end: id(endNode(${r})), props: properties(${r})}`; + return `MATCH (n:DatabaseObject) WHERE n.${idType} = ${idType === 'dbId' ? 'toInteger($id)' : '$id'} + OPTIONAL MATCH (n)-[r1]-${outgoingOnly ? '>' : ''}(m) + OPTIONAL MATCH (m)-[r2:species]->(s) + OPTIONAL MATCH (m)-[r3:regulator|regulatedBy|physicalEntity|crossReference|referenceGene|literatureReference|marker]-(o) + WITH n, collect(DISTINCT m) AS ms, collect(DISTINCT s) AS ss, collect(DISTINCT o) AS os, + collect(DISTINCT r1) + collect(DISTINCT r2) + collect(DISTINCT r3) AS rs + RETURN ${node('n')} AS root, [x IN ms + ss + os | ${node('x')}] AS nodes, + [r IN rs | ${rel('r')}] AS relationships`; +} + /** * Nodes on their own, as SDN maps a query that returns only nodes: each with * its properties and no relationships, one instance per node, so a node that diff --git a/tools/content-node/domain.spec.mjs b/tools/content-node/domain.spec.mjs index 2014fb28..c11f4bc2 100644 --- a/tools/content-node/domain.spec.mjs +++ b/tools/content-node/domain.spec.mjs @@ -80,6 +80,27 @@ describe('mapResult and writeDomain, as SDN and Jackson load and write an object expect(json.output.map((entry) => entry.dbId ?? entry)).toEqual([4, 3]); }); + it('puts the root among the candidates where it first appears, else last, as SDN does', () => { + // A regulation whose reaction is also regulated by another: the root is a + // candidate for the reaction's regulatedBy alongside the other one. + const regulation = ['DatabaseObject', 'Regulation', 'PositiveRegulation']; + const make = (nodes) => ({ + root: node(50, regulation, { displayName: 'root' }), + nodes, + relationships: [rel('regulatedBy', 51, 50), rel('regulatedBy', 51, 52)], + }); + const reaction = node(51, EVENT, { displayName: 'reaction' }); + const other = node(52, regulation, { displayName: 'other' }); + const order = (row) => + write(row).regulatedEntity[0].regulatedBy.map((entry) => entry.dbId ?? entry); + // Not in any list: last. + expect(order(make([reaction, other]))).toEqual([52, 50]); + // In a list before the other: there. + expect(order(make([reaction, node(50, regulation, { displayName: 'root' }), other]))).toEqual([ + 50, 52, + ]); + }); + it('counts a relationship returned twice once, as SDN does', () => { // On two paths, in-depth returns the same relationship twice. `species` // is a List, so a second copy would be a second species. @@ -154,6 +175,36 @@ describe('mapResult and writeDomain, as SDN and Jackson load and write an object expect(write(direct).schemaClass).toBe('Pathway'); }); + it('gives a node the class of the field that reaches it first, depth first, as SDN does', () => { + // Two plain PositiveRegulations replacing a deleted one. The first is + // reached through replacementInstances, an interface (Deletable), and SDN + // takes the first class in its store that fits: a gene expression + // regulation. Mapping it reaches its reaction, whose regulatedBy (a class) + // reaches the second before the deleted one's loop does: the second keeps + // its labels' class. Java writes deleted 9034526's two replacements so. + const regulation = ['DatabaseObject', 'Deletable', 'PositiveRegulation', 'Regulation']; + const row = { + root: node(60, ['DatabaseObject', 'MetaDatabaseObject', 'Deleted'], {}), + nodes: [ + node(61, regulation, { displayName: 'first' }), + node(62, regulation, { displayName: 'second' }), + node(63, EVENT, { displayName: 'reaction' }), + ], + relationships: [ + rel('replacementInstances', 60, 61), + rel('replacementInstances', 60, 62), + rel('regulatedBy', 63, 61), + rel('regulatedBy', 63, 62), + ], + }; + const [first, second] = write(row).replacementInstances; + expect(first.schemaClass).toBe('PositiveGeneExpressionRegulation'); + // Written in full where it was first met, inside the first's reaction. + expect(second).toBe(62); + const [, nested] = first.regulatedEntity[0].regulatedBy; + expect(nested).toMatchObject({ dbId: 62, schemaClass: 'PositiveRegulation' }); + }); + it("keeps a constructor's value where the node has none: moleculeType", () => { const row = { root: node( diff --git a/tools/content-node/service.mjs b/tools/content-node/service.mjs index 999964d0..c0eacbda 100644 --- a/tools/content-node/service.mjs +++ b/tools/content-node/service.mjs @@ -28,6 +28,7 @@ import { findByIdQuery, inDepthQuery, mapNodes, + oldEnhancedQuery, mapResult, rootLabelsQuery, VIEW, @@ -664,6 +665,39 @@ export const endpoints = [ return { status: 200, body, type: 'application/json' }; }), })), + { + // Before /query/{id}/stId and /normalPathway, as Spring ranks its literal + // `enhanced` above their {id}: /query/enhanced/stId is this, with id stId. + path: '/ContentService/data/query/enhanced/{id}', + sample: 'R-HSA-70486', + differs: [/^\/ContentService\/data\/query\/enhanced\/[^/]+: \.url: /], + /** + * An object with its neighbours and some of theirs: graph-core's + * findOldEnhancedObjectById (oldEnhancedQuery), written by the domain + * serialiser in the request's view. The entity popup, the website's + * content pages and the IDG panel read it. + * + * Its relationships go both ways unless QueryObjectController's + * needsIncomingRelationship says not (needsIncoming). The id is read as + * getIdentifier reads it (inDepthId). + */ + handler: async (request) => { + const raw = pathId(rawSegment(request, 5)); + const id = inDepthId(raw); + const missing = () => notFound(request, `Id: ${raw} has not been found in the System`); + if (!id) return missing(); + const outgoingOnly = !(await needsIncoming(raw)); + const [row] = await read(oldEnhancedQuery({ idType: id.idType, outgoingOnly }), { + id: id.id, + }); + if (!row) return missing(); + return { + status: 200, + serialised: writeDomain(mapResult(row), ...writerFor(request)), + type: 'application/json', + }; + }, + }, { path: '/ContentService/data/query/{id}/stId', // A reaction by dbId, so the comparison covers the lookup the site makes. @@ -2697,6 +2731,30 @@ export function inDepthId(raw) { return undefined; } +/** + * QueryObjectController.needsIncomingRelationship, on the id as it came (not + * as getIdentifier reads it): a stable id needs them unless it is a chemical + * (R-ALL-, chebi:); a dbId -- Long.valueOf, so a sign but no padding -- unless + * the object is a Species, Summation, Person, Compartment or SimpleEntity. + * Anything Long.valueOf refuses, or no such object, is Java's caught + * exception: false. + */ +export async function needsIncoming(raw) { + if (raw.startsWith('R') || raw.includes(':')) + return !raw.startsWith('R-ALL-') && !raw.startsWith('chebi:'); + const sign = /^[+-]/.test(raw) ? raw[0] : ''; + const digits = raw.slice(sign.length); + if (!javaNumeric(digits)) return false; + const value = BigInt(`${sign === '-' ? '-' : ''}${asciiDigits(digits)}`); + if (value > 9223372036854775807n || value < -9223372036854775808n) return false; + const [row] = await read( + `MATCH (n:DatabaseObject) WHERE n.dbId = $id + RETURN NOT (n:Species OR n:Summation OR n:Person OR n:Compartment OR n:SimpleEntity) AS needs`, + { id: neo4j.int(value.toString()) } + ); + return row?.needs ?? false; +} + /** * `attributes` as the in-depth controllers read it: Spring joins repeated * values with ',', and an empty one takes the default; String.split(',') diff --git a/tools/content-node/service.spec.mjs b/tools/content-node/service.spec.mjs index a5d2f7a3..99bd0b19 100644 --- a/tools/content-node/service.spec.mjs +++ b/tools/content-node/service.spec.mjs @@ -25,6 +25,7 @@ import { parseHierarchy, pathId, timedOut, + needsIncoming, failureAnswer, inDepthAttributes, inDepthId, @@ -367,6 +368,30 @@ describe('the edges of a path, answered as Tomcat and Spring answer them', () => }); }); +describe('/query/enhanced/{id} ranks above /query/{id}/{attribute}, as in Spring', () => { + it('reads /query/enhanced/stId as the enhanced object with the id stId', async () => { + const response = await fetch(`${base}/ContentService/data/query/enhanced/stId`); + expect(response.status).toBe(404); + expect((await response.json()).messages).toEqual(['Id: stId has not been found in the System']); + }); +}); + +describe("needsIncoming, QueryObjectController's needsIncomingRelationship", () => { + // The branches that need no graph; measured against Java's /query/enhanced. + it('wants them for a stable id, except a chemical', async () => { + expect(await needsIncoming('R-HSA-70486')).toBe(true); + expect(await needsIncoming('REACT_432')).toBe(true); + expect(await needsIncoming('R-ALL-113592')).toBe(false); + expect(await needsIncoming('chebi:15422')).toBe(false); + }); + + it("does not for what Long.valueOf refuses: padding, words, a long's overflow", async () => { + expect(await needsIncoming(' 70486')).toBe(false); + expect(await needsIncoming('foo')).toBe(false); + expect(await needsIncoming('99999999999999999999')).toBe(false); + }); +}); + describe('the in-depth parameters, as the in-depth controllers read them', () => { // Each measured against Java's /data/entity/{id}/in-depth, 15 Oct 2026. it('reads maxDepth as a primitive int with a default', () => {