Port /data/query/enhanced/{id} to node; map results depth first as SDN does

deploy/nginx/common/routes-core.conf

@@ -74,6 +74,17 @@ location = /ContentService/data/species/all {
     proxy_pass http://content_node;
 }
 
+# An object with its neighbours, their species, and their regulations,
+# references and markers one step further: graph-core's
+# findOldEnhancedObjectById, written by the domain serialiser. The entity
+# popup, the website's content pages and the IDG panel read it. Not
+# /enhanced/v2/{id}, which Java still serves. Byte-identical to Java over
+# 6,000 objects of every class.
+location ~ ^/ContentService/data/query/enhanced/[^/]+$ {
+    include /etc/nginx/common/upstream-proxy.conf;
+    proxy_pass http://content_node;
+}
+
 # An entity or an event and everything below it (graph-core's get...InDepth),
 # which the Pathway Browser's object tree reads with JSOG in the
 # nested-aggregated view. Byte-identical to Java over 1,229 requests: 529

projects/website-angular/src/scripts/proxy-order.spec.ts

@@ -70,6 +70,18 @@ describe('the dev server proxy', () => {
     });
   }
 
+  for (const url of [
+    '/ContentService/data/query/enhanced/R-HSA-70486',
+    '/ContentService/data/query/enhanced/70486?includeRef=true',
+    // Java's /query/enhanced/{id} with the id `normalPathway`, not
+    // /query/{id}/normalPathway with the id `enhanced`.
+    '/ContentService/data/query/enhanced/normalPathway',
+  ]) {
+    it(`sends ${url} to the content service`, async () => {
+      expect(await targetOf(url)).toBe(contentNode);
+    });
+  }
+
   for (const url of [
     '/ContentService/data/entity/R-HSA-109796/in-depth',
     '/ContentService/data/entity/109796/in-depth?includeRef=true&view=nested-aggregated',
@@ -171,17 +183,17 @@ describe('the dev server proxy', () => {
     '/ContentService/data/query/70486/displayName',
     '/ContentService/data/query/70486/stIdVersion',
     '/ContentService/data/query/70486/normalPathways',
-    // Java's more specific /query/abridged/{id} and /query/enhanced/{id} take
-    // these, reading the attribute's name as the id.
+    // Java's more specific /query/abridged/{id} takes this, reading the
+    // attribute's name as the id. (/query/enhanced/{id} does the same, and
+    // is node's now: see above.)
     '/ContentService/data/query/abridged/stId',
-    '/ContentService/data/query/enhanced/normalPathway',
     // Java matches attribute names in any case; only the site's spelling is routed.
     '/ContentService/data/query/70486/NORMALPATHWAY',
     // Spring also matches a trailing slash; only the plain object is routed.
     '/ContentService/data/query/70486/',
     // Java's POST /query/ids (a batch of objects); node serves no POST.
     '/ContentService/data/query/ids',
-    '/ContentService/data/query/enhanced/70486',
+    '/ContentService/data/query/enhanced/v2/70486',
     '/ContentService/data/query/abridged/70486',
     '/ContentService/data/person/68285/authoredPathwaysAndMore',
     // Only the ancestors of an event are ported.

projects/website-angular/src/scripts/serve-prod.spec.ts

@@ -315,7 +315,8 @@ describe('serve-prod and the API roots', () => {
     ['/ContentService/data/person/68285/authoredPathwaysAndMore', 'backend'],
     ['/ContentService/data/query/R-HSA-70486', 'content-node'],
     ['/ContentService/data/query/R-HSA-70486?view=nested', 'content-node'],
-    ['/ContentService/data/query/enhanced/R-HSA-70486', 'backend'],
+    ['/ContentService/data/query/enhanced/R-HSA-70486', 'content-node'],
+    ['/ContentService/data/query/enhanced/v2/R-HSA-70486', 'backend'],
     ['/ContentService/data/query/ids', 'backend'],
   ]) {
     it(`sends ${url} to ${to}`, async () => {

proxy.conf.js

@@ -102,6 +102,9 @@ module.exports = {
       },
     ])
   ),
+  // An object with its neighbours and some of theirs (the older enhanced
+  // query). Not /enhanced/v2/, which Java still serves.
+  '^/ContentService/data/query/enhanced/[^/?]+(\\?|$)': contentNode,
   // An entity or an event and everything below it, for the object tree.
   '^/ContentService/data/(entity|event)/[^/?]+/in-depth(\\?|$)': contentNode,
   // A person, and the publications they authored. Not the other attributes

tools/content-node/domain.mjs

@@ -103,50 +103,139 @@ const isA = (schemaClass, target) => supertypes(schemaClass).has(target);
  * one SDN fills fields in (see fill); without it, the nodes' own order.
  */
 export function mapResult({ root, nodes, relationships: returned, order }) {
-  // A relationship returned twice -- on two paths, in-depth -- is one
-  // relationship to SDN: kept where it first appears.
+  // extractRelationships / extractNodes: each relationship and node once, in
+  // the order of first appearance -- the nodes of list columns first, then
+  // single-node columns, so the root sits where it first appears in a list
+  // and otherwise last.
   const seenRelationships = new Set();
-  const relationships = returned.filter(
-    (relationship) =>
-      relationship.id === undefined ||
-      (!seenRelationships.has(relationship.id) && seenRelationships.add(relationship.id))
-  );
-  const entities = new Map();
-  const rootClass = schemaClassOf(root.labels);
-  entities.set(root.id, new Entity(root.id, rootClass, root.props));
-  const byId = new Map(nodes.filter(Boolean).map((node) => [node.id, node]));
-  // A related node's class is chosen for the root's field that reaches it.
-  for (const field of SNAPSHOT().graph[rootClass]?.fields ?? []) {
-    if (field.length === 2) continue;
-    const [, type, direction, target] = field;
-    for (const relationship of relationships) {
-      if (relationship.type !== type) continue;
-      const outgoing = direction === 'OUTGOING';
-      if ((outgoing ? relationship.start : relationship.end) !== root.id) continue;
-      const node = byId.get(outgoing ? relationship.end : relationship.start);
-      if (!node || entities.has(node.id)) continue;
-      entities.set(node.id, new Entity(node.id, classFor(target, node.labels), node.props));
-    }
-  }
-  for (const node of byId.values())
-    if (!entities.has(node.id))
-      entities.set(node.id, new Entity(node.id, schemaClassOf(node.labels), node.props));
-  // The order SDN visits related nodes in: first appearance in the result.
+  const relationships = returned
+    // One without an id (a test's) is its own relationship, under a key of its own.
+    .map((relationship, at) =>
+      relationship.id === undefined ? { ...relationship, id: `#${at}` } : relationship
+    )
+    .filter(
+      (relationship) =>
+        !seenRelationships.has(relationship.id) && seenRelationships.add(relationship.id)
+    );
+  const listed = nodes.filter(Boolean);
+  const byId = new Map();
+  for (const node of [...listed, root]) if (!byId.has(node.id)) byId.set(node.id, node);
   const position = new Map();
-  for (const id of [root.id, ...(order ?? nodes.filter(Boolean).map((node) => node.id))])
+  for (const id of [...(order ?? listed.map((node) => node.id)), root.id])
     if (!position.has(id)) position.set(id, position.size);
-  // Each relationship under both of its ends, so a field finds its own at once.
-  const byEnd = new Map();
-  const add = (key, relationship) => {
-    if (!byEnd.has(key)) byEnd.set(key, []);
-    byEnd.get(key).push(relationship);
+  const state = {
+    byId,
+    position,
+    relationships,
+    removed: new Set(),
+    processed: new Map(),
+    known: new Map(),
+    byEnd: new Map(),
   };
   for (const relationship of relationships) {
-    add(`OUTGOING ${relationship.type} ${relationship.start}`, relationship);
-    add(`INCOMING ${relationship.type} ${relationship.end}`, relationship);
+    for (const key of [
+      `OUTGOING ${relationship.type} ${relationship.start}`,
+      `INCOMING ${relationship.type} ${relationship.end}`,
+    ]) {
+      if (!state.byEnd.has(key)) state.byEnd.set(key, []);
+      state.byEnd.get(key).push(relationship);
+    }
+  }
+  return map(state, root, undefined);
+}
+
+/**
+ * DefaultNeo4jEntityConverter.map and createInstanceOfRelationships: the
+ * root as an entity, and everything its fields reach.
+ *
+ * A node becomes an entity once, its class decided by the field that first
+ * reaches it (classFor) -- so which field reaches a node first matters, and
+ * SDN goes depth first: an entity met in a field has its own fields filled
+ * before that field goes on to the next. Done here with a stack of frames, in
+ * that order, rather than by recursion, which a deep in-depth answer -- an
+ * object reached through thousands of others -- would run out of stack on.
+ *
+ * Each field: the candidates are the result's nodes that carry the target's
+ * label, in result order; for each, the field's relationships to it, each
+ * mapped as it is met. A relationship is counted per direction each time a
+ * field uses it, and once one has been used twice in the same direction it
+ * leaves the result for every field after -- SDN's guard against cycles,
+ * kept for fidelity: on Reactome's model only PhysicalEntity has fields
+ * sharing a type and direction (three `regulator` INCOMING), a node can match
+ * at most two, and each owner is mapped once, so it does not fire. Not
+ * reproduced: SDN gathers a list column's relationships into a HashSet, so two
+ * parallel edges of one type between the same nodes would come in its hash
+ * order, not the result's; whether the graph has any is unchecked. Filling node
+ * by node rather than relationship by relationship is visible when one node
+ * is reached twice: an entity that is both a reaction's input (order 2) and
+ * its output (order 1) is the first output, because it was met among the
+ * inputs; Java writes R-CEL-156804's reaction that way.
+ */
+function map(state, node, target) {
+  const frames = [];
+  const make = (from, to) => {
+    const schemaClass = to ? classFor(to, from.labels) : schemaClassOf(from.labels);
+    const entity = new Entity(from.id, schemaClass, from.props);
+    // Known before its fields are filled, so a field leading back finds it.
+    state.known.set(from.id, entity);
+    const graph = SNAPSHOT().graph[schemaClass];
+    if (graph) {
+      for (const field of graph.fields)
+        if (field.length === 2)
+          entity.fields[field[0]] = entity.properties[field[1]] ?? graph.defaults?.[field[0]];
+      frames.push({ entity, fields: graph.fields.filter((field) => field.length > 2), at: 0 });
+    }
+    return entity;
+  };
+  const root = make(node, target);
+  while (frames.length) {
+    const frame = frames[frames.length - 1];
+    if (!frame.matching) {
+      if (frame.at === frame.fields.length) {
+        frames.pop();
+        continue;
+      }
+      frame.matching = matchingFor(state, frame.entity, frame.fields[frame.at]);
+      frame.next = 0;
+      frame.found = [];
+    }
+    const [name, , direction, target, collection, relationClass] = frame.fields[frame.at];
+    if (frame.next === frame.matching.length) {
+      frame.entity.fields[name] = collect(frame.found, collection, relationClass);
+      frame.matching = undefined;
+      frame.at += 1;
+      continue;
+    }
+    const relationship = frame.matching[frame.next];
+    frame.next += 1;
+    const key = `${direction} ${relationship.id}`;
+    const used = state.processed.get(key) ?? 0;
+    if (used === 2) state.removed.add(relationship.id);
+    else state.processed.set(key, used + 1);
+    const otherId = direction === 'OUTGOING' ? relationship.end : relationship.start;
+    // A new entity pushes its frame: its fields are filled before this one goes on.
+    const other =
+      otherId === frame.entity.id
+        ? frame.entity
+        : (state.known.get(otherId) ?? make(state.byId.get(otherId), target));
+    frame.found.push(
+      relationClass ? new Relation(relationClass, other, relationship.props) : other
+    );
   }
-  for (const entity of entities.values()) fill(entity, entities, byEnd, position);
-  return entities.get(root.id);
+  return root;
+}
+
+/** One field's relationships, in candidate order (see map). */
+function matchingFor(state, entity, [, type, direction, target]) {
+  const outgoing = direction === 'OUTGOING';
+  const otherEnd = (relationship) => (outgoing ? relationship.end : relationship.start);
+  return (
+    (state.byEnd.get(`${direction} ${type} ${entity.id}`) ?? [])
+      .filter((relationship) => !state.removed.has(relationship.id))
+      .filter((relationship) => state.byId.get(otherEnd(relationship))?.labels.includes(target))
+      // Candidate order; stable, so relationships to one node keep theirs.
+      .sort((a, b) => state.position.get(otherEnd(a)) - state.position.get(otherEnd(b)))
+  );
 }
 
 /**
@@ -174,42 +263,10 @@ function classFor(target, labels) {
   return best ?? schemaClassOf(labels);
 }
 
-/**
- * One entity's fields. A relationship field is filled node by node, in the
- * order the nodes first appear in the result, and only then relationship by
- * relationship -- not in the relationships' own order. The difference shows
- * when one node is reached twice: an entity that is both a reaction's input
- * (order 2) and its output (order 1) is the *first* output, because it was
- * met among the inputs; Java writes R-CEL-156804's reaction that way.
- */
-function fill(entity, entities, byEnd, position) {
-  const graph = SNAPSHOT().graph[entity.schemaClass];
-  if (!graph) return;
-  for (const field of graph.fields) {
-    if (field.length === 2) {
-      entity.fields[field[0]] = entity.properties[field[1]] ?? graph.defaults?.[field[0]];
-      continue;
-    }
-    const [name, type, direction, target, collection, relationClass] = field;
-    const outgoing = direction === 'OUTGOING';
-    const otherEnd = (relationship) => (outgoing ? relationship.end : relationship.start);
-    // Stable: relationships to the same node keep their own order.
-    const matching = [...(byEnd.get(`${direction} ${type} ${entity.id}`) ?? [])].sort(
-      (a, b) => position.get(otherEnd(a)) - position.get(otherEnd(b))
-    );
-    const found = [];
-    for (const relationship of matching) {
-      const other = entities.get(otherEnd(relationship));
-      if (!other || !isA(other.schemaClass, target)) continue;
-      found.push(relationClass ? new Relation(relationClass, other, relationship.props) : other);
-    }
-    entity.fields[name] = collect(found, collection, relationClass);
-  }
-}
-
 /** The field's collection as SDN creates it: ArrayList, LinkedHashSet, TreeSet. */
 function collect(found, collection, relationClass) {
-  if (collection === '') return found.length ? found[found.length - 1] : undefined;
+  // A single value is the first found (createInstanceOfRelationships: value.get(0)).
+  if (collection === '') return found.length ? found[0] : undefined;
   if (collection === 'List') return found;
   // Has.equals compares elements; DatabaseObject.equals, the node.
   const key = relationClass ? (item) => item.element : (item) => item;
@@ -511,6 +568,30 @@ export function inDepthQuery({ label, follow, idType, maxDepth, attributes }) {
             [r IN relationships | ${rel('r')}] AS relationships`;
 }
 
+/**
+ * graph-core's findOldEnhancedObjectById: the object, everything it is
+ * related to (outgoing only, or both ways), those objects' species, and their
+ * regulations, physical entities, references and markers one step further.
+ *
+ * Java collects every row's nodes, repeats and all, which for a busy object
+ * is the same nodes many times over. Only the order of first appearance
+ * matters to the mapping (fill), and collecting each list DISTINCT keeps that
+ * order: a node first appears where it first appears in its first list.
+ */
+export function oldEnhancedQuery({ idType, outgoingOnly }) {
+  const node = (n) => `{id: id(${n}), labels: labels(${n}), props: properties(${n})}`;
+  const rel = (r) =>
+    `{id: id(${r}), type: type(${r}), start: id(startNode(${r})), end: id(endNode(${r})), props: properties(${r})}`;
+  return `MATCH (n:DatabaseObject) WHERE n.${idType} = ${idType === 'dbId' ? 'toInteger($id)' : '$id'}
+     OPTIONAL MATCH (n)-[r1]-${outgoingOnly ? '>' : ''}(m)
+     OPTIONAL MATCH (m)-[r2:species]->(s)
+     OPTIONAL MATCH (m)-[r3:regulator|regulatedBy|physicalEntity|crossReference|referenceGene|literatureReference|marker]-(o)
+     WITH n, collect(DISTINCT m) AS ms, collect(DISTINCT s) AS ss, collect(DISTINCT o) AS os,
+          collect(DISTINCT r1) + collect(DISTINCT r2) + collect(DISTINCT r3) AS rs
+     RETURN ${node('n')} AS root, [x IN ms + ss + os | ${node('x')}] AS nodes,
+            [r IN rs | ${rel('r')}] AS relationships`;
+}
+
 /**
  * Nodes on their own, as SDN maps a query that returns only nodes: each with
  * its properties and no relationships, one instance per node, so a node that

tools/content-node/domain.spec.mjs

@@ -80,6 +80,27 @@ describe('mapResult and writeDomain, as SDN and Jackson load and write an object
     expect(json.output.map((entry) => entry.dbId ?? entry)).toEqual([4, 3]);
   });
 
+  it('puts the root among the candidates where it first appears, else last, as SDN does', () => {
+    // A regulation whose reaction is also regulated by another: the root is a
+    // candidate for the reaction's regulatedBy alongside the other one.
+    const regulation = ['DatabaseObject', 'Regulation', 'PositiveRegulation'];
+    const make = (nodes) => ({
+      root: node(50, regulation, { displayName: 'root' }),
+      nodes,
+      relationships: [rel('regulatedBy', 51, 50), rel('regulatedBy', 51, 52)],
+    });
+    const reaction = node(51, EVENT, { displayName: 'reaction' });
+    const other = node(52, regulation, { displayName: 'other' });
+    const order = (row) =>
+      write(row).regulatedEntity[0].regulatedBy.map((entry) => entry.dbId ?? entry);
+    // Not in any list: last.
+    expect(order(make([reaction, other]))).toEqual([52, 50]);
+    // In a list before the other: there.
+    expect(order(make([reaction, node(50, regulation, { displayName: 'root' }), other]))).toEqual([
+      50, 52,
+    ]);
+  });
+
   it('counts a relationship returned twice once, as SDN does', () => {
     // On two paths, in-depth returns the same relationship twice. `species`
     // is a List, so a second copy would be a second species.
@@ -154,6 +175,36 @@ describe('mapResult and writeDomain, as SDN and Jackson load and write an object
     expect(write(direct).schemaClass).toBe('Pathway');
   });
 
+  it('gives a node the class of the field that reaches it first, depth first, as SDN does', () => {
+    // Two plain PositiveRegulations replacing a deleted one. The first is
+    // reached through replacementInstances, an interface (Deletable), and SDN
+    // takes the first class in its store that fits: a gene expression
+    // regulation. Mapping it reaches its reaction, whose regulatedBy (a class)
+    // reaches the second before the deleted one's loop does: the second keeps
+    // its labels' class. Java writes deleted 9034526's two replacements so.
+    const regulation = ['DatabaseObject', 'Deletable', 'PositiveRegulation', 'Regulation'];
+    const row = {
+      root: node(60, ['DatabaseObject', 'MetaDatabaseObject', 'Deleted'], {}),
+      nodes: [
+        node(61, regulation, { displayName: 'first' }),
+        node(62, regulation, { displayName: 'second' }),
+        node(63, EVENT, { displayName: 'reaction' }),
+      ],
+      relationships: [
+        rel('replacementInstances', 60, 61),
+        rel('replacementInstances', 60, 62),
+        rel('regulatedBy', 63, 61),
+        rel('regulatedBy', 63, 62),
+      ],
+    };
+    const [first, second] = write(row).replacementInstances;
+    expect(first.schemaClass).toBe('PositiveGeneExpressionRegulation');
+    // Written in full where it was first met, inside the first's reaction.
+    expect(second).toBe(62);
+    const [, nested] = first.regulatedEntity[0].regulatedBy;
+    expect(nested).toMatchObject({ dbId: 62, schemaClass: 'PositiveRegulation' });
+  });
+
   it("keeps a constructor's value where the node has none: moleculeType", () => {
     const row = {
       root: node(