From 9ee816643c5c6dd57bbd6b249f6872922e0a33dd Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Sat, 3 Aug 2024 12:08:36 +0800 Subject: [PATCH 01/17] enlarge bytes limit --- .../backend/serializer/BinarySerializer.java | 4 ++-- .../hugegraph/backend/serializer/BytesBuffer.java | 13 +++++++++---- .../org/apache/hugegraph/structure/HugeElement.java | 3 +-- .../java/org/apache/hugegraph/task/HugeTask.java | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java index 37a7e9a9ca..0dd4c9d3ee 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java @@ -1312,9 +1312,9 @@ private Id readId(byte[] value) { } private byte[] writeIds(Collection ids) { - E.checkState(ids.size() <= BytesBuffer.UINT16_MAX, + E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES, "The number of properties of vertex/edge label " + - "can't exceed '%s'", BytesBuffer.UINT16_MAX); + "can't exceed '%s'", BytesBuffer.MAX_PROPERTIES); int size = 2; for (Id id : ids) { size += (1 + id.length()); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index f293dd2873..74d78b3df6 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -52,10 +52,12 @@ public final class BytesBuffer extends OutputStream { public static final int FLOAT_LEN = Float.BYTES; public static final int DOUBLE_LEN = Double.BYTES; public static final int BLOB_LEN = 4; + public static final int BYTES_LEN = 5; public static final int UINT8_MAX = ((byte) -1) & 0xff; public static final int UINT16_MAX = ((short) -1) & 0xffff; public static final long UINT32_MAX = (-1) & 0xffffffffL; + public static final int INT32_MAX = Integer.MAX_VALUE; // NOTE: +1 to let code 0 represent length 1 public static final int ID_LEN_MASK = 0x7f; @@ -64,8 +66,11 @@ public final class BytesBuffer extends OutputStream { public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; - public static final int STRING_LEN_MAX = UINT16_MAX; + public static final long BLOB_LEN_MAX = 1 * Bytes.GB; + public static final long BYTES_LEN_MAX = INT32_MAX; + + public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; // The value must be in range [8, ID_LEN_MAX] public static final int INDEX_HASH_ID_THRESHOLD = 32; @@ -288,10 +293,10 @@ public double readDouble() { } public BytesBuffer writeBytes(byte[] bytes) { - E.checkArgument(bytes.length <= UINT16_MAX, + E.checkArgument(bytes.length <= BYTES_LEN_MAX, "The max length of bytes is %s, but got %s", - UINT16_MAX, bytes.length); - require(SHORT_LEN + bytes.length); + BYTES_LEN_MAX, bytes.length); + require(BYTES_LEN + bytes.length); this.writeVInt(bytes.length); this.write(bytes); return this; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java index 137e623d86..adb12f31bc 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java @@ -55,7 +55,6 @@ public abstract class HugeElement implements Element, GraphType, Idfiable, Compa private static final MutableIntObjectMap> EMPTY_MAP = CollectionFactory.newIntObjectMap(); - private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; private final HugeGraph graph; private MutableIntObjectMap> properties; @@ -279,7 +278,7 @@ public HugeProperty setProperty(HugeProperty prop) { PropertyKey pkey = prop.propertyKey(); E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) || - this.properties.size() < MAX_PROPERTIES, + this.properties.size() < BytesBuffer.MAX_PROPERTIES, "Exceeded the maximum number of properties"); return this.properties.put(intFromId(pkey.id()), prop); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java index bfd79f6f22..95ef506f5a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java @@ -725,7 +725,7 @@ private void checkPropertySize(String property, String propertyName) { } private void checkPropertySize(int propertyLength, String propertyName) { - long propertyLimit = BytesBuffer.STRING_LEN_MAX; + long propertyLimit = BytesBuffer.MAX_PROPERTIES; HugeGraph graph = this.scheduler().graph(); if (propertyName.equals(P.INPUT)) { propertyLimit = graph.option(CoreOptions.TASK_INPUT_SIZE_LIMIT); From 6effaafa7248cc397da443a16ddf04f0df945248 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 6 Aug 2024 17:45:23 +0800 Subject: [PATCH 02/17] encode string id length with 2 bits by default --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index 74d78b3df6..fee2a3e613 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -634,7 +634,7 @@ public Object readProperty(DataType dataType) { } public BytesBuffer writeId(Id id) { - return this.writeId(id, false); + return this.writeId(id, true); } public BytesBuffer writeId(Id id, boolean big) { @@ -684,7 +684,7 @@ public BytesBuffer writeId(Id id, boolean big) { } public Id readId() { - return this.readId(false); + return this.readId(true); } public Id readId(boolean big) { From a24ee3dfcfda50aa5ea3cf8c5b224bbdf9f8adb3 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 6 Aug 2024 17:52:27 +0800 Subject: [PATCH 03/17] add comment support user-defined configuration --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index fee2a3e613..dbbdb492e1 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -59,6 +59,7 @@ public final class BytesBuffer extends OutputStream { public static final long UINT32_MAX = (-1) & 0xffffffffL; public static final int INT32_MAX = Integer.MAX_VALUE; + // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 public static final int ID_LEN_MASK = 0x7f; public static final int ID_LEN_MAX = 0x7f + 1; // 128 @@ -67,6 +68,7 @@ public final class BytesBuffer extends OutputStream { public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; + // TODO: support user-defined configuration public static final long BLOB_LEN_MAX = 1 * Bytes.GB; public static final long BYTES_LEN_MAX = INT32_MAX; From cc165a54d08059f9fecd4284fdfe5d5cbc894f95 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 6 Aug 2024 19:46:23 +0800 Subject: [PATCH 04/17] fix UT --- .../apache/hugegraph/unit/id/IdUtilTest.java | 24 ++++++++++++++++--- .../unit/serializer/BytesBufferTest.java | 22 +++++++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java index fa582322ab..25f53dbc7c 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java @@ -60,7 +60,7 @@ public void testWriteReadBinString() { Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = IdGenerator.of("123"); - bytes = ByteBuffer.wrap(genBytes("82313233")); + bytes = ByteBuffer.wrap(genBytes("8002313233")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); @@ -71,12 +71,12 @@ public void testWriteReadBinString() { Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = EdgeId.parse("S1>2>3>L4"); - bytes = ByteBuffer.wrap(genBytes("7e803182080233000804")); + bytes = ByteBuffer.wrap(genBytes("7e80003182080233000804")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = EdgeId.parse("S1111>2222>3>L4444"); - bytes = ByteBuffer.wrap(genBytes("7e8331313131821808ae330018115c")); + bytes = ByteBuffer.wrap(genBytes("7e800331313131821808ae330018115c")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); @@ -154,4 +154,22 @@ private byte[] genBytes(String string) { } return bytes; } + + /** + * Converts a byte array to a hexadecimal string. + * + * @param bytes the byte array to convert + * @return the hexadecimal string representation of the byte array + */ + private String bytesToHex(byte[] bytes) { + StringBuilder hexString = new StringBuilder(); + for (byte b : bytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + hexString.append('0'); // pad with leading zero if needed + } + hexString.append(hex); + } + return hexString.toString(); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java index 9072652908..963c2af806 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java @@ -79,14 +79,14 @@ public void testWrap() { @Test public void testStringId() { Id id = IdGenerator.of("abc"); - byte[] bytes = new byte[]{(byte) 0x82, 97, 98, 99}; + byte[] bytes = new byte[]{(byte) 0x80, 2, 97, 98, 99}; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(4) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); id = IdGenerator.of("abcd"); - bytes = new byte[]{(byte) 0x83, 97, 98, 99, 100}; + bytes = new byte[]{(byte) 0x80, 3, 97, 98, 99, 100}; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(5) .writeId(id).bytes()); @@ -96,29 +96,31 @@ public void testStringId() { @Test public void testStringIdWithBigSize() { Id id = IdGenerator.of(genString(127)); - byte[] bytes = genBytes(128); - bytes[0] = (byte) 0xfe; + byte[] bytes = genBytes(129); + bytes[0] = (byte) 0x80; + bytes[1] = (byte) 0x7e; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); id = IdGenerator.of(genString(128)); - bytes = genBytes(129); - bytes[0] = (byte) 0xff; + bytes = genBytes(130); + bytes[0] = (byte) 0x80; + bytes[1] = (byte) 0x7f; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(129))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32769))); }, e -> { - Assert.assertContains("Id max length is 128, but got 129", + Assert.assertContains("Big id max length is 32768, but got 32769", e.getMessage()); }); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(130))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32770))); }, e -> { - Assert.assertContains("Id max length is 128, but got 130", + Assert.assertContains("Big id max length is 32768, but got 32770", e.getMessage()); }); } From 2da4cb3912855abb3fc1e39f926cd0c0dff1f5d2 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 7 Aug 2024 23:51:41 +0800 Subject: [PATCH 05/17] sync with internal --- .../hugegraph/backend/cache/OffheapCache.java | 6 +-- .../backend/serializer/BytesBuffer.java | 36 +++++-------- .../apache/hugegraph/structure/HugeEdge.java | 4 +- .../apache/hugegraph/core/EdgeCoreTest.java | 5 +- .../apache/hugegraph/unit/id/IdUtilTest.java | 6 +-- .../unit/serializer/BytesBufferTest.java | 53 ++++++++++--------- 6 files changed, 52 insertions(+), 58 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index d641c8276f..dd2cfce552 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -162,19 +162,19 @@ private class IdSerializer implements CacheSerializer { @Override public Id deserialize(ByteBuffer input) { - return BytesBuffer.wrap(input).readId(true); + return BytesBuffer.wrap(input).readId(); } @Override public void serialize(Id id, ByteBuffer output) { - BytesBuffer.wrap(output).writeId(id, true); + BytesBuffer.wrap(output).writeId(id); } @Override public int serializedSize(Id id) { // NOTE: return size must be == actual bytes to write return BytesBuffer.allocate(id.length() + 2) - .writeId(id, true).position(); + .writeId(id).position(); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index dbbdb492e1..2dd24c7588 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -61,9 +61,7 @@ public final class BytesBuffer extends OutputStream { // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 - public static final int ID_LEN_MASK = 0x7f; - public static final int ID_LEN_MAX = 0x7f + 1; // 128 - public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768 + public static final int ID_LEN_MAX = 0x3fff + 1; // 16384 public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; @@ -636,10 +634,6 @@ public Object readProperty(DataType dataType) { } public BytesBuffer writeId(Id id) { - return this.writeId(id, true); - } - - public BytesBuffer writeId(Id id, boolean big) { switch (id.type()) { case LONG: // Number Id @@ -663,20 +657,18 @@ public BytesBuffer writeId(Id id, boolean big) { bytes = id.asBytes(); int len = bytes.length; E.checkArgument(len > 0, "Can't write empty id"); - if (!big) { - E.checkArgument(len <= ID_LEN_MAX, - "Id max length is %s, but got %s {%s}", - ID_LEN_MAX, len, id); - len -= 1; // mapping [1, 128] to [0, 127] + E.checkArgument(len <= ID_LEN_MAX, + "Big id max length is %s, but got %s {%s}", + ID_LEN_MAX, len, id); + len -= 1; // mapping [1, 16384] to [0, 16383] + if (len <= 0x3f) { + // If length is <= 63, use a single byte with the highest bit set to 1 this.writeUInt8(len | 0x80); } else { - E.checkArgument(len <= BIG_ID_LEN_MAX, - "Big id max length is %s, but got %s {%s}", - BIG_ID_LEN_MAX, len, id); - len -= 1; int high = len >> 8; int low = len & 0xff; - this.writeUInt8(high | 0x80); + // Write high 8 bits with highest two bits set to 11 + this.writeUInt8(high | 0xc0); this.writeUInt8(low); } this.write(bytes); @@ -686,10 +678,6 @@ public BytesBuffer writeId(Id id, boolean big) { } public Id readId() { - return this.readId(true); - } - - public Id readId(boolean big) { byte b = this.read(); boolean number = (b & 0x80) == 0; if (number) { @@ -705,13 +693,13 @@ public Id readId(boolean big) { } } else { // String Id - int len = b & ID_LEN_MASK; - if (big) { + int len = b & 0x3f; // Take the lowest 6 bits as part of the length + if ((b & 0x40) != 0) { // If the 7th bit is set, length information spans 2 bytes int high = len << 8; int low = this.readUInt8(); len = high + low; } - len += 1; // restore [0, 127] to [1, 128] + len += 1; // restore [0, 16383] to [1, 16384] byte[] id = this.read(len); return IdGenerator.of(id, IdType.STRING); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java index 53b8a54933..9ea7f52f30 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java @@ -147,9 +147,9 @@ public void assignId() { if (this.fresh()) { int len = this.id.length(); - E.checkArgument(len <= BytesBuffer.BIG_ID_LEN_MAX, + E.checkArgument(len <= BytesBuffer.ID_LEN_MAX, "The max length of edge id is %s, but got %s {%s}", - BytesBuffer.BIG_ID_LEN_MAX, len, this.id); + BytesBuffer.ID_LEN_MAX, len, this.id); } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index c7353e4a7c..38bddec1f4 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -470,12 +470,13 @@ public void testAddEdgeWithLargeSortkey() { Vertex book = graph.addVertex(T.label, "book", "name", "Test-Book-1"); Assert.assertThrows(IllegalArgumentException.class, () -> { - final int LEN = BytesBuffer.BIG_ID_LEN_MAX; + final int LEN = BytesBuffer.ID_LEN_MAX; String largeTime = "{large-time}" + new String(new byte[LEN]); james.addEdge("write", book, "time", largeTime); graph.tx().commit(); }, e -> { - Assert.assertContains("The max length of edge id is 32768", + Assert.assertContains(String.format("The max length of edge id is %s", + BytesBuffer.ID_LEN_MAX), e.getMessage()); }); } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java index 25f53dbc7c..fe9925b2ba 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java @@ -60,7 +60,7 @@ public void testWriteReadBinString() { Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = IdGenerator.of("123"); - bytes = ByteBuffer.wrap(genBytes("8002313233")); + bytes = ByteBuffer.wrap(genBytes("82313233")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); @@ -71,12 +71,12 @@ public void testWriteReadBinString() { Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = EdgeId.parse("S1>2>3>L4"); - bytes = ByteBuffer.wrap(genBytes("7e80003182080233000804")); + bytes = ByteBuffer.wrap(genBytes("7e803182080233000804")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); id = EdgeId.parse("S1111>2222>3>L4444"); - bytes = ByteBuffer.wrap(genBytes("7e800331313131821808ae330018115c")); + bytes = ByteBuffer.wrap(genBytes("7e8331313131821808ae330018115c")); Assert.assertEquals(bytes, IdUtil.writeBinString(id)); Assert.assertEquals(id, IdUtil.readBinString(bytes)); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java index 963c2af806..fb25a5b255 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java @@ -79,14 +79,14 @@ public void testWrap() { @Test public void testStringId() { Id id = IdGenerator.of("abc"); - byte[] bytes = new byte[]{(byte) 0x80, 2, 97, 98, 99}; + byte[] bytes = new byte[]{(byte) 0x82, 97, 98, 99}; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(4) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); id = IdGenerator.of("abcd"); - bytes = new byte[]{(byte) 0x80, 3, 97, 98, 99, 100}; + bytes = new byte[]{(byte) 0x83, 97, 98, 99, 100}; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(5) .writeId(id).bytes()); @@ -97,7 +97,7 @@ public void testStringId() { public void testStringIdWithBigSize() { Id id = IdGenerator.of(genString(127)); byte[] bytes = genBytes(129); - bytes[0] = (byte) 0x80; + bytes[0] = (byte) 0xc0; bytes[1] = (byte) 0x7e; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); @@ -105,22 +105,26 @@ public void testStringIdWithBigSize() { id = IdGenerator.of(genString(128)); bytes = genBytes(130); - bytes[0] = (byte) 0x80; + bytes[0] = (byte) 0xc0; bytes[1] = (byte) 0x7f; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32769))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 1))); }, e -> { - Assert.assertContains("Big id max length is 32768, but got 32769", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 1), e.getMessage()); }); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32770))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 2))); }, e -> { - Assert.assertContains("Big id max length is 32768, but got 32770", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 2), e.getMessage()); }); } @@ -129,33 +133,34 @@ public void testStringIdWithBigSize() { public void testStringBigId() { Id id = IdGenerator.of(genString(128)); byte[] bytes = genBytes(130); - bytes[0] = (byte) 0x80; + bytes[0] = (byte) 0xc0; bytes[1] = (byte) 0x7f; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); - id = IdGenerator.of(genString(32512)); - bytes = genBytes(32514); - bytes[0] = (byte) 0xfe; - bytes[1] = (byte) 0xff; + id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX - 1)); + bytes = genBytes(BytesBuffer.ID_LEN_MAX + 1); + bytes[0] = (byte) 0xff; + bytes[1] = (byte) 0xfe; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); - id = IdGenerator.of(genString(32768)); - bytes = genBytes(32770); + id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX)); + bytes = genBytes(BytesBuffer.ID_LEN_MAX + 2); bytes[0] = (byte) 0xff; bytes[1] = (byte) 0xff; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32769)), - true); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 1))); }, e -> { - Assert.assertContains("Big id max length is 32768, but got 32769", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 1), e.getMessage()); }); } From f8f383d23d4b6fb47af18841a80347009cb599a8 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 7 Aug 2024 23:54:10 +0800 Subject: [PATCH 06/17] remove useless import --- .../src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index 38bddec1f4..d466d1d782 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -75,7 +75,6 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; import org.junit.Assume; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableList; From 822bf170aa2176bea3a3d219b51651b115935d14 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 00:47:23 +0800 Subject: [PATCH 07/17] fixup --- .../java/org/apache/hugegraph/core/VertexCoreTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index c9a83ddc15..a329de3afb 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -44,6 +44,7 @@ import org.apache.hugegraph.backend.query.Condition; import org.apache.hugegraph.backend.query.ConditionQuery; import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.serializer.BytesBuffer; import org.apache.hugegraph.backend.store.BackendTable; import org.apache.hugegraph.backend.store.Shard; import org.apache.hugegraph.backend.tx.GraphTransaction; @@ -1039,10 +1040,10 @@ public void testAddVertexWithCustomizeStringIdStrategyWithoutValidId() { "name", "marko", "age", 18, "city", "Beijing"); }); - // Expect id length <= 128 + // Expect id length <= BytesBuffer.ID_LEN_MAX Assert.assertThrows(IllegalArgumentException.class, () -> { - String largeId = new String(new byte[128]) + "."; - assert largeId.length() == 129; + String largeId = new String(new byte[BytesBuffer.ID_LEN_MAX]) + "."; + assert largeId.length() == BytesBuffer.ID_LEN_MAX + 1; graph.addVertex(T.label, "programmer", T.id, largeId, "name", "marko", "age", 18, "city", "Beijing"); }); From 61dbd1481fee39c56878ac75baf6e1470f1026db Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 19:04:32 +0800 Subject: [PATCH 08/17] enlarge ENTRY_SIZE --- .../main/java/org/apache/hugegraph/unit/cache/CacheTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java index 63837c8aab..537c0087fd 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java @@ -138,7 +138,7 @@ protected void checkNotInCache(Cache cache, Id id) { public static class OffheapCacheTest extends CacheTest { - private static final long ENTRY_SIZE = 40L; + private static final long ENTRY_SIZE = 200L; private final HugeGraph graph = Mockito.mock(HugeGraph.class); @Override From a8e4c57b5256f8a81903381708d2d98750210924 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 20:05:52 +0800 Subject: [PATCH 09/17] try fix --- .../org/apache/hugegraph/backend/cache/OffheapCache.java | 5 +++-- .../java/org/apache/hugegraph/unit/cache/CacheTest.java | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index dd2cfce552..27627e3b47 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -57,12 +57,13 @@ public class OffheapCache extends AbstractCache { public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) { // NOTE: capacity unit is bytes, the super capacity expect elements size super(capacity); - long capacityInBytes = capacity * (avgEntryBytes + 64L); + int segments = Runtime.getRuntime().availableProcessors() * 2; + long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L); if (capacityInBytes <= 0L) { capacityInBytes = 1L; } this.graph = graph; - this.cache = this.builder().capacity(capacityInBytes).build(); + this.cache = this.builder().capacity(capacityInBytes).segmentCount(segments).build(); this.serializer = new BinarySerializer(); } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java index 537c0087fd..b1903f8837 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java @@ -138,7 +138,7 @@ protected void checkNotInCache(Cache cache, Id id) { public static class OffheapCacheTest extends CacheTest { - private static final long ENTRY_SIZE = 200L; + private static final long ENTRY_SIZE = 40L; private final HugeGraph graph = Mockito.mock(HugeGraph.class); @Override @@ -325,7 +325,7 @@ public void testUpdateAndGetWithDataType() { @Test public void testUpdateAndGetWithSameSizeAndCapacity() { - int limit = 40; + int limit = 1000; Cache cache = newCache(limit); Map map = new LimitMap(limit); @@ -451,7 +451,7 @@ public void testSize() { @Test public void testSizeWithReachCapacity() { - int limit = 20; + int limit = 1000; Cache cache = newCache(limit); Map map = new LimitMap(limit); From d8eb24f108f9ee3e108a946c6b343ea1600451d4 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 20:24:41 +0800 Subject: [PATCH 10/17] fixed segments --- .../org/apache/hugegraph/backend/cache/OffheapCache.java | 5 ++++- .../main/java/org/apache/hugegraph/unit/cache/CacheTest.java | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index 27627e3b47..f829cda4b1 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -55,9 +55,12 @@ public class OffheapCache extends AbstractCache { private final AbstractSerializer serializer; public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) { + this(graph, capacity, avgEntryBytes, Runtime.getRuntime().availableProcessors() * 2); + } + + public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) { // NOTE: capacity unit is bytes, the super capacity expect elements size super(capacity); - int segments = Runtime.getRuntime().availableProcessors() * 2; long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L); if (capacityInBytes <= 0L) { capacityInBytes = 1L; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java index b1903f8837..d9bd4cf953 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java @@ -139,6 +139,7 @@ protected void checkNotInCache(Cache cache, Id id) { public static class OffheapCacheTest extends CacheTest { private static final long ENTRY_SIZE = 40L; + private static final int SEGMENTS = 4; private final HugeGraph graph = Mockito.mock(HugeGraph.class); @Override @@ -148,7 +149,7 @@ protected Cache newCache() { @Override protected Cache newCache(long capacity) { - return new OffheapCache(this.graph(), capacity, ENTRY_SIZE); + return new OffheapCache(this.graph(), capacity, ENTRY_SIZE, SEGMENTS); } @Override From e3b6cd670baadfaa1147c31447d3ccf696787e09 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 20:25:51 +0800 Subject: [PATCH 11/17] minor improve --- .../java/org/apache/hugegraph/backend/cache/OffheapCache.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index f829cda4b1..d9a099eeb2 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -177,8 +177,7 @@ public void serialize(Id id, ByteBuffer output) { @Override public int serializedSize(Id id) { // NOTE: return size must be == actual bytes to write - return BytesBuffer.allocate(id.length() + 2) - .writeId(id).position(); + return BytesBuffer.allocate(id.length() + 2).writeId(id).position(); } } From 7fbf05a49dda7c9c75b1472107ec81b2c3280680 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 23:07:18 +0800 Subject: [PATCH 12/17] sync with internal --- .../apache/hugegraph/backend/serializer/BytesBuffer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index 2dd24c7588..f824274e2d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -51,8 +51,8 @@ public final class BytesBuffer extends OutputStream { public static final int CHAR_LEN = Character.BYTES; public static final int FLOAT_LEN = Float.BYTES; public static final int DOUBLE_LEN = Double.BYTES; - public static final int BLOB_LEN = 4; - public static final int BYTES_LEN = 5; + public static final int BYTES_LEN = 4; + public static final int BLOB_LEN = 5; public static final int UINT8_MAX = ((byte) -1) & 0xff; public static final int UINT16_MAX = ((short) -1) & 0xffff; @@ -67,8 +67,8 @@ public final class BytesBuffer extends OutputStream { public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; // TODO: support user-defined configuration + public static final long BYTES_LEN_MAX = 10 * Bytes.MB; public static final long BLOB_LEN_MAX = 1 * Bytes.GB; - public static final long BYTES_LEN_MAX = INT32_MAX; public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; From 7893b7f81bf05bf2fab375c07c4f486c1fe87a3f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 8 Aug 2024 23:09:51 +0800 Subject: [PATCH 13/17] remove useless var --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index f824274e2d..850de22354 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -57,7 +57,6 @@ public final class BytesBuffer extends OutputStream { public static final int UINT8_MAX = ((byte) -1) & 0xff; public static final int UINT16_MAX = ((short) -1) & 0xffff; public static final long UINT32_MAX = (-1) & 0xffffffffL; - public static final int INT32_MAX = Integer.MAX_VALUE; // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 From c059de635d3623396a82ce7b217a657bc4eb7a84 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Aug 2024 23:43:59 +0800 Subject: [PATCH 14/17] refact: disable strict option & tiny fix --- .asf.yaml | 4 +- .../hugegraph/api/filter/AccessLogFilter.java | 3 +- .../apache/hugegraph/core/GraphManager.java | 6 +- .../hugegraph/backend/cache/OffheapCache.java | 17 ++--- .../org/apache/hugegraph/backend/id/Id.java | 3 +- .../backend/serializer/BytesBuffer.java | 70 +++++++------------ 6 files changed, 38 insertions(+), 65 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 6c78530425..93063dc7d0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,8 +34,8 @@ github: protected_branches: master: required_status_checks: - # strict means "Require branches to be up-to-date before merging". (TODO: turnoff when branch is stable) - strict: true + # strict means "Require PR to be up-to-date before merging". (enable when branch unstable) + strict: false # contexts are the names of checks that must pass (now only enable the basic check) contexts: - Analyze (java) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index d429db4d9b..ae53686467 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.net.URI; -import org.apache.hugegraph.auth.HugeAuthenticator; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.core.GraphManager; @@ -124,7 +123,7 @@ public void filter(ContainerRequestContext requestContext, GraphManager manager = managerProvider.get(); // TODO: transfer Authorizer if we need after. if (manager.requireAuthentication()) { - manager.unauthorize(requestContext.getSecurityContext()); + manager.unauthorized(requestContext.getSecurityContext()); } } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index 37939c2019..5cba0a0f0f 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -265,7 +265,7 @@ public HugeAuthenticator.User authenticate(Map credentials) return this.authenticator().authenticate(credentials); } - public void unauthorize(SecurityContext context) { + public void unauthorized(SecurityContext context) { this.authenticator().unauthorize(context); } @@ -515,7 +515,7 @@ private boolean supportRoleElection() { private void addMetrics(HugeConfig config) { final MetricManager metric = MetricManager.INSTANCE; - // Force to add server reporter + // Force to add a server reporter ServerReporter reporter = ServerReporter.instance(metric.getRegistry()); reporter.start(60L, TimeUnit.SECONDS); @@ -610,7 +610,7 @@ private void dropGraph(HugeGraph graph) { /* * Will fill graph instance into HugeFactory.graphs after - * GraphFactory.open() succeed, remove it when graph drop + * GraphFactory.open() succeed, remove it when the graph drops */ HugeFactory.remove(graph); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index d9a099eeb2..7ed4efcd66 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -59,7 +59,7 @@ public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) { } public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) { - // NOTE: capacity unit is bytes, the super capacity expect elements size + // NOTE: capacity unit is bytes, the super capacity expects elements' size super(capacity); long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L); if (capacityInBytes <= 0L) { @@ -232,7 +232,6 @@ public ByteBuffer asBuffer() { } BytesBuffer buffer = BytesBuffer.allocate(64 * listSize); - // May fail to serialize and throw exception here this.serialize(this.value, buffer); @@ -279,8 +278,7 @@ private Object deserialize(BytesBuffer buffer) { } } - private void serializeList(BytesBuffer buffer, - Collection list) { + private void serializeList(BytesBuffer buffer, Collection list) { // Write list buffer.writeVInt(list.size()); for (Object i : list) { @@ -298,8 +296,7 @@ private List deserializeList(BytesBuffer buffer) { return list; } - private void serializeElement(BytesBuffer buffer, - ValueType type, Object value) { + private void serializeElement(BytesBuffer buffer, ValueType type, Object value) { E.checkNotNull(value, "serialize value"); BackendEntry entry; if (type == ValueType.VERTEX) { @@ -335,14 +332,12 @@ private Object deserializeElement(ValueType type, BytesBuffer buffer) { } private HugeException unsupported(ValueType type) { - throw new HugeException( - "Unsupported deserialize type: %s", type); + throw new HugeException("Unsupported deserialize type: %s", type); } private HugeException unsupported(Object value) { - throw new HugeException( - "Unsupported type of serialize value: '%s'(%s)", - value, value.getClass()); + throw new HugeException("Unsupported type of serialize value: '%s'(%s)", + value, value.getClass()); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java index b0e5613495..077aa43dbb 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java @@ -67,8 +67,7 @@ public char prefix() { } public static IdType valueOfPrefix(String id) { - E.checkArgument(id != null && !id.isEmpty(), - "Invalid id '%s'", id); + E.checkArgument(id != null && !id.isEmpty(), "Invalid id '%s'", id); switch (id.charAt(0)) { case 'L': return IdType.LONG; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index 2dd24c7588..42cbc93568 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -92,8 +92,7 @@ public BytesBuffer() { public BytesBuffer(int capacity) { E.checkArgument(capacity <= MAX_BUFFER_CAPACITY, - "Capacity exceeds max buffer capacity: %s", - MAX_BUFFER_CAPACITY); + "Capacity exceeds max buffer capacity: %s", MAX_BUFFER_CAPACITY); this.buffer = ByteBuffer.allocate(capacity); this.resize = true; } @@ -172,8 +171,7 @@ private void require(int size) { // Extra capacity as buffer int newCapacity = size + this.buffer.limit() + DEFAULT_CAPACITY; E.checkArgument(newCapacity <= MAX_BUFFER_CAPACITY, - "Capacity exceeds max buffer capacity: %s", - MAX_BUFFER_CAPACITY); + "Capacity exceeds max buffer capacity: %s", MAX_BUFFER_CAPACITY); ByteBuffer newBuffer = ByteBuffer.allocate(newCapacity); this.buffer.flip(); newBuffer.put(this.buffer); @@ -293,8 +291,8 @@ public double readDouble() { } public BytesBuffer writeBytes(byte[] bytes) { - E.checkArgument(bytes.length <= BYTES_LEN_MAX, - "The max length of bytes is %s, but got %s", + // TODO: update to 10MB.. + E.checkArgument(bytes.length <= BYTES_LEN_MAX, "The max length of bytes is %s, but got %s", BYTES_LEN_MAX, bytes.length); require(BYTES_LEN + bytes.length); this.writeVInt(bytes.length); @@ -309,8 +307,8 @@ public byte[] readBytes() { } public BytesBuffer writeBigBytes(byte[] bytes) { - E.checkArgument(bytes.length <= BLOB_LEN_MAX, - "The max length of bytes is %s, but got %s", + // TODO: note the max blob size should be 128MB(due to MAX_BUFFER_CAPACITY) + E.checkArgument(bytes.length <= BLOB_LEN_MAX, "The max length of bytes is %s, but got %s", BLOB_LEN_MAX, bytes.length); require(BLOB_LEN + bytes.length); this.writeVInt(bytes.length); @@ -351,9 +349,7 @@ public BytesBuffer writeStringWithEnding(String value) { assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) : "Invalid UTF8 bytes: " + value; if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { - E.checkArgument(false, - "Can't contains byte '0x00' in string: '%s'", - value); + E.checkArgument(false, "Can't contains byte '0x00' in string: '%s'", value); } this.write(bytes); } @@ -435,8 +431,7 @@ public BytesBuffer writeVInt(int value) { public int readVInt() { byte leading = this.read(); - E.checkArgument(leading != 0x80, - "Unexpected varint with leading byte '0x%s'", + E.checkArgument(leading != 0x80, "Unexpected varint with leading byte '0x%s'", Bytes.toHex(leading)); int value = leading & 0x7f; if (leading >= 0) { @@ -455,8 +450,7 @@ public int readVInt() { } } - E.checkArgument(i < 5, - "Unexpected varint %s with too many bytes(%s)", + E.checkArgument(i < 5, "Unexpected varint %s with too many bytes(%s)", value, i + 1); E.checkArgument(i < 4 || (leading & 0x70) == 0, "Unexpected varint %s with leading byte '0x%s'", @@ -499,8 +493,7 @@ public BytesBuffer writeVLong(long value) { public long readVLong() { byte leading = this.read(); - E.checkArgument(leading != 0x80, - "Unexpected varlong with leading byte '0x%s'", + E.checkArgument(leading != 0x80, "Unexpected varlong with leading byte '0x%s'", Bytes.toHex(leading)); long value = leading & 0x7fL; if (leading >= 0) { @@ -519,8 +512,7 @@ public long readVLong() { } } - E.checkArgument(i < 10, - "Unexpected varlong %s with too many bytes(%s)", + E.checkArgument(i < 10, "Unexpected varlong %s with too many bytes(%s)", value, i + 1); E.checkArgument(i < 9 || (leading & 0x7e) == 0, "Unexpected varlong %s with leading byte '0x%s'", @@ -586,8 +578,7 @@ public void writeProperty(DataType dataType, Object value) { this.writeString((String) value); break; case BLOB: - byte[] bytes = value instanceof byte[] ? - (byte[]) value : ((Blob) value).bytes(); + byte[] bytes = value instanceof byte[] ? (byte[]) value : ((Blob) value).bytes(); this.writeBigBytes(bytes); break; case UUID: @@ -597,6 +588,7 @@ public void writeProperty(DataType dataType, Object value) { this.writeLong(uuid.getLeastSignificantBits()); break; default: + // TODO: replace Kryo with Fury (https://github.com/apache/fury) this.writeBytes(KryoUtil.toKryoWithType(value)); break; } @@ -629,6 +621,7 @@ public Object readProperty(DataType dataType) { case UUID: return new UUID(this.readLong(), this.readLong()); default: + // TODO: replace Kryo with Apache Fury return KryoUtil.fromKryoWithType(this.readBytes()); } } @@ -641,24 +634,23 @@ public BytesBuffer writeId(Id id) { this.writeNumber(value); break; case UUID: - // UUID Id + // UUID ID byte[] bytes = id.asBytes(); assert bytes.length == Id.UUID_LENGTH; this.writeUInt8(0x7f); // 0b01111111 means UUID this.write(bytes); break; case EDGE: - // Edge Id + // Edge ID this.writeUInt8(0x7e); // 0b01111110 means EdgeId this.writeEdgeId(id); break; default: - // String Id + // String Id (VertexID) bytes = id.asBytes(); int len = bytes.length; E.checkArgument(len > 0, "Can't write empty id"); - E.checkArgument(len <= ID_LEN_MAX, - "Big id max length is %s, but got %s {%s}", + E.checkArgument(len <= ID_LEN_MAX, "Big id max length is %s, but got %s {%s}", ID_LEN_MAX, len, id); len -= 1; // mapping [1, 16384] to [0, 16383] if (len <= 0x3f) { @@ -734,12 +726,11 @@ public BytesBuffer writeIndexId(Id id, HugeType type, boolean withEnding) { this.write(bytes); if (type.isStringIndex()) { if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { - // Not allow STRING_ENDING_BYTE exist in string index id + // Not allow STRING_ENDING_BYTE to exist in string index id E.checkArgument(false, "The %s type index id can't contains " + - "byte '0x%s', but got: 0x%s", type, - Bytes.toHex(STRING_ENDING_BYTE), - Bytes.toHex(bytes)); + "byte '0x%s', but got: 0x%s", + type, Bytes.toHex(STRING_ENDING_BYTE), Bytes.toHex(bytes)); } if (withEnding) { this.writeStringWithEnding(""); @@ -789,24 +780,15 @@ public BinaryId parseId(HugeType type, boolean enablePartition) { return new BinaryId(bytes, id); } - /** - * 解析 olap id - * @param type - * @param isOlap - * @return - */ public BinaryId parseOlapId(HugeType type, boolean isOlap) { if (type.isIndex()) { return this.readIndexId(type); } // Parse id from bytes int start = this.buffer.position(); - /** - * OLAP - * {PropertyKey}{VertexId} - */ + // OLAP {PropertyKey}{VertexId} if (isOlap) { - // 先 read olap property id + // Read olap property id first Id pkId = this.readId(); } Id id = this.readId(); @@ -885,8 +867,7 @@ private void writeNumber(long val) { } private long readNumber(byte b) { - E.checkArgument((b & 0x80) == 0, - "Not a number type with prefix byte '0x%s'", + E.checkArgument((b & 0x80) == 0, "Not a number type with prefix byte '0x%s'", Bytes.toHex(b)); // Parse the kind from byte 0kkksxxx int kind = b >>> 4; @@ -943,8 +924,7 @@ private byte[] readBytesWithEnding() { break; } } - E.checkArgument(foundEnding, "Not found ending '0x%s'", - Bytes.toHex(STRING_ENDING_BYTE)); + E.checkArgument(foundEnding, "Not found ending '0x%s'", Bytes.toHex(STRING_ENDING_BYTE)); int end = this.buffer.position() - 1; int len = end - start; if (len <= 0) { From df58a484993e55b13695decc59b074b0cd45bb03 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 9 Aug 2024 00:37:01 +0800 Subject: [PATCH 15/17] intro EID_LEN_MAX --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 3 ++- .../main/java/org/apache/hugegraph/structure/HugeEdge.java | 4 ++-- .../src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index fc1342628e..e9ff371a27 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -60,7 +60,8 @@ public final class BytesBuffer extends OutputStream { // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 - public static final int ID_LEN_MAX = 0x3fff + 1; // 16384 + public static final int ID_LEN_MAX = 0x3fff + 1; // 16KB + public static final int EID_LEN_MAX = 0x1ffff + 1; // 128KB public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java index 9ea7f52f30..f38c2b067f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java @@ -147,9 +147,9 @@ public void assignId() { if (this.fresh()) { int len = this.id.length(); - E.checkArgument(len <= BytesBuffer.ID_LEN_MAX, + E.checkArgument(len <= BytesBuffer.EID_LEN_MAX, "The max length of edge id is %s, but got %s {%s}", - BytesBuffer.ID_LEN_MAX, len, this.id); + BytesBuffer.EID_LEN_MAX, len, this.id); } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index d466d1d782..85f9eab5ff 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -469,13 +469,13 @@ public void testAddEdgeWithLargeSortkey() { Vertex book = graph.addVertex(T.label, "book", "name", "Test-Book-1"); Assert.assertThrows(IllegalArgumentException.class, () -> { - final int LEN = BytesBuffer.ID_LEN_MAX; + final int LEN = BytesBuffer.EID_LEN_MAX; String largeTime = "{large-time}" + new String(new byte[LEN]); james.addEdge("write", book, "time", largeTime); graph.tx().commit(); }, e -> { Assert.assertContains(String.format("The max length of edge id is %s", - BytesBuffer.ID_LEN_MAX), + BytesBuffer.EID_LEN_MAX), e.getMessage()); }); } From f2a750db6e60d4ec97b6da8a8da3c113a68b0509 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 9 Aug 2024 00:40:17 +0800 Subject: [PATCH 16/17] minor improve --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index e9ff371a27..cf74d4a389 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -291,7 +291,6 @@ public double readDouble() { } public BytesBuffer writeBytes(byte[] bytes) { - // TODO: update to 10MB.. E.checkArgument(bytes.length <= BYTES_LEN_MAX, "The max length of bytes is %s, but got %s", BYTES_LEN_MAX, bytes.length); require(BYTES_LEN + bytes.length); @@ -307,7 +306,7 @@ public byte[] readBytes() { } public BytesBuffer writeBigBytes(byte[] bytes) { - // TODO: note the max blob size should be 128MB(due to MAX_BUFFER_CAPACITY) + // TODO: note the max blob size should be 128MB (due to MAX_BUFFER_CAPACITY) E.checkArgument(bytes.length <= BLOB_LEN_MAX, "The max length of bytes is %s, but got %s", BLOB_LEN_MAX, bytes.length); require(BLOB_LEN + bytes.length); @@ -621,7 +620,7 @@ public Object readProperty(DataType dataType) { case UUID: return new UUID(this.readLong(), this.readLong()); default: - // TODO: replace Kryo with Apache Fury + // TODO: replace Kryo with Fury (https://github.com/apache/fury) return KryoUtil.fromKryoWithType(this.readBytes()); } } From d81b7d0dfdeb7c3afb462e555b84e3e9cddf2908 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 9 Aug 2024 00:44:04 +0800 Subject: [PATCH 17/17] minor improve --- .../org/apache/hugegraph/backend/serializer/BytesBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index cf74d4a389..40aae34557 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -61,7 +61,7 @@ public final class BytesBuffer extends OutputStream { // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 public static final int ID_LEN_MAX = 0x3fff + 1; // 16KB - public static final int EID_LEN_MAX = 0x1ffff + 1; // 128KB + public static final int EID_LEN_MAX = 64 * 1024; public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff;