Merge pull request #10018 from neo-technology/aioob-degrees-louvain

Mats-SX · web-flow · commit 8e854a7d7d5c · 2025-02-06T09:27:39.000+01:00
Add validation during relationships scanning which excludes relationships to nodes that were not loaded during node scanning
diff --git a/core/src/integrationTest/java/org/neo4j/gds/core/GraphLoaderTest.java b/core/src/integrationTest/java/org/neo4j/gds/core/GraphLoaderTest.java
@@ -127,8 +127,8 @@ void shouldLogProgressWithNativeLoading() {
             .build()
             .graph();
 
-        log.assertContainsMessage(TestLog.DEBUG, "Loading :: Nodes :: Store Scan :: Imported 3 records and 1 properties");
-        log.assertContainsMessage(TestLog.DEBUG, "Loading :: Relationships :: Store Scan :: Imported 4 records and 0 properties");
+        log.assertContainsMessage(TestLog.INFO, "Loading :: Nodes :: Store Scan :: Imported 3 records and 1 properties");
+        log.assertContainsMessage(TestLog.INFO, "Loading :: Relationships :: Store Scan :: Imported 4 records and 0 properties");
     }
 
     @Test
@@ -163,13 +163,13 @@ void shouldTrackProgressWithNativeLoading() {
             )
             .doesNotContain("Loading :: Nodes :: Property Index Scan :: Start");
 
-        assertThat(log.getMessages(TestLog.DEBUG))
+        assertThat(log.getMessages(TestLog.INFO))
             .extracting(removingThreadId())
             .contains("Loading :: Nodes :: Store Scan :: Start using NodeCursorBasedScanner")
             .contains("Loading :: Relationships :: Store Scan :: Start using RelationshipScanCursorBasedScanner");
 
-        log.assertContainsMessage(TestLog.DEBUG, "Loading :: Nodes :: Store Scan :: Imported 3 records and 1 properties");
-        log.assertContainsMessage(TestLog.DEBUG, "Loading :: Relationships :: Store Scan :: Imported 4 records and 0 properties");
+        log.assertContainsMessage(TestLog.INFO, "Loading :: Nodes :: Store Scan :: Imported 3 records and 1 properties");
+        log.assertContainsMessage(TestLog.INFO, "Loading :: Relationships :: Store Scan :: Imported 4 records and 0 properties");
     }
 
     @Test
diff --git a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java
@@ -59,10 +59,50 @@ public interface AdjacencyCompressor extends AutoCloseable {
     void close();
 
     interface ValueMapper {
+        int INVALID_ID = -1;
+
         /**
          * A mapper to transform values before compressing them.
+         * May return {@link #INVALID_ID} to indicate that the value should be skipped.
          * Implementations must be thread-safe
          */
         long map(long value);
+
+        default ValueMapper andThen(ValueMapper other) {
+            return new AndThen(this, other);
+        }
+
+        enum Identity implements AdjacencyCompressor.ValueMapper {
+            INSTANCE {
+                @Override
+                public long map(long value) {
+                    return value;
+                }
+
+                @Override
+                public ValueMapper andThen(ValueMapper other) {
+                    return other;
+                }
+            }
+        }
+
+        final class AndThen implements ValueMapper {
+            private final ValueMapper first;
+            private final ValueMapper second;
+
+            AndThen(ValueMapper first, ValueMapper second) {
+                this.first = first;
+                this.second = second;
+            }
+
+            @Override
+            public long map(long value) {
+                var firstMapped = this.first.map(value);
+                if (firstMapped == INVALID_ID) {
+                    return INVALID_ID;
+                }
+                return this.second.map(firstMapped);
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressorFactory.java b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressorFactory.java
@@ -44,4 +44,6 @@ public interface AdjacencyCompressorFactory {
      * @return the final adjacency list, together with any number of properties, if any.
      */
     AdjacencyListsWithProperties build(boolean allowReordering);
+
+    boolean validateNode(long nodeId);
 }
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/AbstractAdjacencyCompressorFactory.java b/core/src/main/java/org/neo4j/gds/core/compression/common/AbstractAdjacencyCompressorFactory.java
@@ -26,9 +26,9 @@
 import org.neo4j.gds.api.compress.AdjacencyListBuilder;
 import org.neo4j.gds.api.compress.AdjacencyListsWithProperties;
 import org.neo4j.gds.api.compress.ImmutableAdjacencyListsWithProperties;
-import org.neo4j.gds.core.Aggregation;
 import org.neo4j.gds.collections.ha.HugeIntArray;
 import org.neo4j.gds.collections.ha.HugeLongArray;
+import org.neo4j.gds.core.Aggregation;
 
 import java.util.concurrent.atomic.LongAdder;
 import java.util.function.LongSupplier;
@@ -42,6 +42,10 @@ public abstract class AbstractAdjacencyCompressorFactory<TARGET_PAGE, PROPERTY_P
     private final Aggregation[] aggregations;
     private final LongAdder relationshipCounter;
 
+    // set during `init`, used to validate nodes against this value since it is also
+    // used to size the adjacency lists
+    private long nodeCount = -1;
+
     private HugeIntArray adjacencyDegrees;
     private HugeLongArray adjacencyOffsets;
     private HugeLongArray propertyOffsets;
@@ -67,6 +71,7 @@ public void init() {
         this.adjacencyDegrees = HugeIntArray.newArray(nodeCount);
         this.adjacencyOffsets = HugeLongArray.newArray(nodeCount);
         this.propertyOffsets = HugeLongArray.newArray(nodeCount);
+        this.nodeCount = nodeCount;
     }
 
     @Override
@@ -81,6 +86,12 @@ public LongAdder relationshipCounter() {
         return relationshipCounter;
     }
 
+    @Override
+    public boolean validateNode(long nodeId) {
+        assert this.nodeCount != -1 : "Called validateNode before init";
+        return nodeId < this.nodeCount;
+    }
+
     @Override
     public AdjacencyListsWithProperties build(boolean allowReordering) {
         var builder = ImmutableAdjacencyListsWithProperties
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java b/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java
@@ -50,19 +50,18 @@ public static void zigZagUncompressFrom(
         AdjacencyCompressor.ValueMapper mapper
     ) {
         into.ensureCapacity(compressedValues);
-        zigZagUncompressFrom(into.buffer, targets, compressedValues, limit, mapper);
-        into.length = compressedValues;
+        into.length = zigZagUncompressFrom(into.buffer, targets, compressedValues, limit, mapper);
     }
 
-    public static void zigZagUncompressFrom(
+    public static int zigZagUncompressFrom(
         long[] into,
         byte[] targets,
         int compressedValues,
         int limit,
         AdjacencyCompressor.ValueMapper mapper
     ) {
         assert into.length >= compressedValues;
-        zigZagUncompress(targets, limit, into, mapper);
+        return zigZagUncompress(targets, limit, into, mapper);
     }
 
     public static int applyDeltaEncoding(LongArrayBuffer data, Aggregation aggregation) {
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java b/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java
@@ -20,17 +20,11 @@
 package org.neo4j.gds.core.compression.common;
 
 import org.neo4j.gds.api.compress.AdjacencyCompressor;
+import org.neo4j.gds.api.compress.AdjacencyCompressor.ValueMapper.Identity;
 
-public final class ZigZagLongDecoding {
+import static org.neo4j.gds.api.compress.AdjacencyCompressor.ValueMapper.INVALID_ID;
 
-    public enum Identity implements AdjacencyCompressor.ValueMapper {
-        INSTANCE {
-            @Override
-            public long map(long value) {
-                return value;
-            }
-        }
-    }
+public final class ZigZagLongDecoding {
 
     public static int zigZagUncompress(byte[] compressedData, int length, long[] uncompressedData) {
         return zigZagUncompress(compressedData, length, uncompressedData, Identity.INSTANCE);
@@ -49,7 +43,11 @@ static int zigZagUncompress(
             value += (input & 127L) << shift;
             if ((input & 128L) == 128L) {
                 startValue += ((value >>> 1L) ^ -(value & 1L));
-                uncompressedData[into++] = mapper.map(startValue);
+                var mappedId = mapper.map(startValue);
+                if (mappedId != INVALID_ID) {
+                    uncompressedData[into++] = mappedId;
+                }
+
                 value = 0L;
                 shift = 0;
             } else {
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/mixed/MixedCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/mixed/MixedCompressor.java
@@ -120,7 +120,6 @@ static class Factory implements AdjacencyCompressorFactory {
         private final AdjacencyCompressorFactory packedCompressorFactory;
         private final AdjacencyCompressorFactory vlongCompressorFactory;
 
-
         Factory(
             LongSupplier nodeCountSupplier,
             LongAdder relationshipCounter,
@@ -194,6 +193,11 @@ public AdjacencyListsWithProperties build(boolean allowReordering) {
                 .build();
         }
 
+        @Override
+        public boolean validateNode(long nodeId) {
+            return this.vlongCompressorFactory.validateNode(nodeId) && this.packedCompressorFactory.validateNode(nodeId);
+        }
+
         private static MemoryInfo mergeMemoryInfo(
             MemoryInfo packed,
             MemoryInfo vlong
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java
@@ -29,12 +29,12 @@
 import org.neo4j.gds.api.compress.AdjacencyListBuilder;
 import org.neo4j.gds.api.compress.AdjacencyListBuilderFactory;
 import org.neo4j.gds.api.compress.ModifiableSlice;
+import org.neo4j.gds.collections.ha.HugeIntArray;
+import org.neo4j.gds.collections.ha.HugeLongArray;
 import org.neo4j.gds.core.Aggregation;
 import org.neo4j.gds.core.compression.common.AbstractAdjacencyCompressorFactory;
 import org.neo4j.gds.core.compression.common.AdjacencyCompression;
 import org.neo4j.gds.core.compression.common.MemoryTracker;
-import org.neo4j.gds.collections.ha.HugeIntArray;
-import org.neo4j.gds.collections.ha.HugeLongArray;
 import org.neo4j.gds.utils.GdsFeatureToggles;
 
 import java.util.Arrays;
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java
@@ -29,12 +29,12 @@
 import org.neo4j.gds.api.compress.AdjacencyListBuilder;
 import org.neo4j.gds.api.compress.AdjacencyListBuilderFactory;
 import org.neo4j.gds.api.compress.ModifiableSlice;
+import org.neo4j.gds.collections.ha.HugeIntArray;
+import org.neo4j.gds.collections.ha.HugeLongArray;
 import org.neo4j.gds.core.Aggregation;
 import org.neo4j.gds.core.compression.common.AbstractAdjacencyCompressorFactory;
 import org.neo4j.gds.core.compression.common.MemoryTracker;
 import org.neo4j.gds.core.utils.AscendingLongComparator;
-import org.neo4j.gds.collections.ha.HugeIntArray;
-import org.neo4j.gds.collections.ha.HugeLongArray;
 
 import java.util.Arrays;
 import java.util.function.LongSupplier;
diff --git a/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java
@@ -28,13 +28,13 @@
 import org.neo4j.gds.api.compress.AdjacencyListBuilder;
 import org.neo4j.gds.api.compress.AdjacencyListBuilderFactory;
 import org.neo4j.gds.api.compress.ModifiableSlice;
+import org.neo4j.gds.collections.ha.HugeIntArray;
+import org.neo4j.gds.collections.ha.HugeLongArray;
 import org.neo4j.gds.core.Aggregation;
 import org.neo4j.gds.core.compression.common.AbstractAdjacencyCompressorFactory;
 import org.neo4j.gds.core.compression.common.AdjacencyCompression;
 import org.neo4j.gds.core.compression.common.MemoryTracker;
 import org.neo4j.gds.core.compression.common.VarLongEncoding;
-import org.neo4j.gds.collections.ha.HugeIntArray;
-import org.neo4j.gds.collections.ha.HugeLongArray;
 
 import java.util.Arrays;
 import java.util.function.LongSupplier;
diff --git a/core/src/main/java/org/neo4j/gds/core/loading/AdjacencyBuffer.java b/core/src/main/java/org/neo4j/gds/core/loading/AdjacencyBuffer.java
@@ -26,11 +26,11 @@
 import org.jetbrains.annotations.Nullable;
 import org.neo4j.gds.RelationshipType;
 import org.neo4j.gds.api.compress.AdjacencyCompressor;
+import org.neo4j.gds.api.compress.AdjacencyCompressor.ValueMapper.Identity;
 import org.neo4j.gds.api.compress.AdjacencyCompressorFactory;
 import org.neo4j.gds.api.compress.LongArrayBuffer;
 import org.neo4j.gds.core.Aggregation;
 import org.neo4j.gds.core.compression.common.AdjacencyCompression;
-import org.neo4j.gds.core.compression.common.ZigZagLongDecoding;
 import org.neo4j.gds.core.concurrency.Concurrency;
 import org.neo4j.gds.mem.MemoryEstimation;
 import org.neo4j.gds.mem.MemoryEstimations;
@@ -248,7 +248,7 @@ Collection<AdjacencyListBuilderTask> adjacencyListBuilderTasks(
                 adjacencyCompressorFactory,
                 chunkedAdjacencyLists[page],
                 relationshipCounter,
-                mapper.orElse(ZigZagLongDecoding.Identity.INSTANCE),
+                mapper.orElse(Identity.INSTANCE),
                 drainCountConsumer.orElse(n -> {})
             ));
         }
@@ -305,32 +305,48 @@ public static final class AdjacencyListBuilderTask implements Runnable {
 
         @Override
         public void run() {
+            var mapper = this.valueMapper.andThen(this::validateNode);
             try (var compressor = adjacencyCompressorFactory.createCompressor()) {
                 var buffer = new LongArrayBuffer();
                 var importedRelationships = new MutableLong(0L);
                 chunkedAdjacencyLists.consume((localId, targets, properties, compressedByteSize, numberOfCompressedTargets) -> {
                     var sourceNodeId = this.paging.sourceNodeId(localId, this.page);
-                    var nodeId = valueMapper.map(sourceNodeId);
+                    var nodeId = mapper.map(sourceNodeId);
+                    if (nodeId == AdjacencyCompressor.ValueMapper.INVALID_ID) {
+                        return;
+                    }
 
                     AdjacencyCompression.zigZagUncompressFrom(
                         buffer,
                         targets,
                         numberOfCompressedTargets,
                         compressedByteSize,
-                        valueMapper
+                        mapper
                     );
 
+                    if (buffer.length == 0) {
+                        // special case: we skipped a relationship because it pointed to or from a node that we didn't load
+                        // this can result in some nodes that should not get an adjacencylist at all
+                        return;
+                    }
+
                     importedRelationships.add(compressor.compress(
                         nodeId,
                         buffer.buffer,
                         properties,
-                        numberOfCompressedTargets
+                        buffer.length
                     ));
                 });
                 relationshipCounter.add(importedRelationships.longValue());
                 drainCountConsumer.accept(importedRelationships.longValue());
             }
         }
+
+        long validateNode(long nodeId) {
+            return (this.adjacencyCompressorFactory.validateNode(nodeId))
+                ? nodeId
+                : AdjacencyCompressor.ValueMapper.INVALID_ID;
+        }
     }
 
     private static final class PagingWithKnownPageSize implements AdjacencyBufferPaging {
diff --git a/core/src/test/java/org/neo4j/gds/core/loading/AdjacencyBufferTest.java b/core/src/test/java/org/neo4j/gds/core/loading/AdjacencyBufferTest.java
diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java
diff --git a/native-projection/src/main/java/org/neo4j/gds/projection/ScanningRecordsImporter.java b/native-projection/src/main/java/org/neo4j/gds/projection/ScanningRecordsImporter.java
diff --git a/test-utils/src/main/java/org/neo4j/gds/core/loading/AdjacencyListBuilderBaseTest.java b/test-utils/src/main/java/org/neo4j/gds/core/loading/AdjacencyListBuilderBaseTest.java

Original file line number	Diff line number	Diff line change
`@@ -44,4 +44,6 @@ public interface AdjacencyCompressorFactory {`
`44`	`44`	`* @return the final adjacency list, together with any number of properties, if any.`
`45`	`45`	`*/`
`46`	`46`	`AdjacencyListsWithProperties build(boolean allowReordering);`
	`47`	`+`
	`48`	`+ boolean validateNode(long nodeId);`
`47`	`49`	`}`