Skip to content

Commit cf8322e

Browse files
Ngalstyan4Ngalstyan4
committed
Add support and tests for install-time schema relocation
Our extension cannot be fully relocatable since its objects span multiple schemas (we have _lantern_internal schema) It can be (and as of this commit) is install-time relocatable so it can be installed on a custom schema, different from the default one. See 38.17.2 in the link below for details https://www.postgresql.org/docs/current/extend-extensions.html#EXTEND-EXTENSIONS-RELOCATION Note: access methods in postgres are not schema-qualified so they are always available globally, regardless of the schema on which the extension is installed https://www.postgresql.org/docs/current/sql-create-access-method.html
1 parent 00c5b41 commit cf8322e

File tree

5 files changed

+173
-6
lines changed

5 files changed

+173
-6
lines changed

‎cmake/lantern.control.template

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
comment = 'LanternDB: Fast vector embedding processing in Postgres'
1+
comment = 'Lantern: Fast vector embedding processing in Postgres'
22
default_version = '@LANTERNDB_VERSION@'
33
module_pathname = '$libdir/lantern'
4-
relocatable = true
4+
relocatable = false

‎sql/lantern.sql

+4-3
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ CREATE OPERATOR <-> (
2929
COMMUTATOR = '<->'
3030
);
3131

32+
CREATE SCHEMA _lantern_internal;
3233
-- operator classes
33-
CREATE OR REPLACE FUNCTION _create_ldb_operator_classes(access_method_name TEXT) RETURNS BOOLEAN AS $$
34+
CREATE OR REPLACE FUNCTION _lantern_internal._create_ldb_operator_classes(access_method_name TEXT) RETURNS BOOLEAN AS $$
3435
DECLARE
3536
dist_l2sq_ops TEXT;
3637
dist_cos_ops TEXT;
@@ -107,13 +108,13 @@ BEGIN
107108

108109

109110
IF hnsw_am_exists THEN
110-
PERFORM _create_ldb_operator_classes('lantern_hnsw');
111+
PERFORM _lantern_internal._create_ldb_operator_classes('lantern_hnsw');
111112
RAISE WARNING 'Access method(index type) "hnsw" already exists. Creating lantern_hnsw access method';
112113
ELSE
113114
-- create access method
114115
CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnsw_handler;
115116
COMMENT ON ACCESS METHOD hnsw IS 'LanternDB access method for vector embeddings, based on the hnsw algorithm';
116-
PERFORM _create_ldb_operator_classes('hnsw');
117+
PERFORM _lantern_internal._create_ldb_operator_classes('hnsw');
117118
END IF;
118119
END;
119120
$BODY$

‎test/expected/ext_relocation.out

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
\ir utils/small_world_array.sql
2+
CREATE TABLE small_world (
3+
id VARCHAR(3),
4+
b BOOLEAN,
5+
v REAL[3]
6+
);
7+
INSERT INTO small_world (id, b, v) VALUES
8+
('000', TRUE, '{0,0,0}'),
9+
('001', TRUE, '{0,0,1}'),
10+
('010', FALSE, '{0,1,0}'),
11+
('011', TRUE, '{0,1,1}'),
12+
('100', FALSE, '{1,0,0}'),
13+
('101', FALSE, '{1,0,1}'),
14+
('110', FALSE, '{1,1,0}'),
15+
('111', TRUE, '{1,1,1}');
16+
DROP EXTENSION lantern;
17+
\set ON_ERROR_STOP off
18+
-- make sure the extension was dropped.
19+
CREATE INDEX ON small_world USING hnsw (v) WITH (dim=3);
20+
ERROR: access method "hnsw" does not exist
21+
\set ON_ERROR_STOP on
22+
-- test creating lantern on different schemas
23+
CREATE SCHEMA schema1;
24+
CREATE SCHEMA schema2;
25+
CREATE EXTENSION lantern WITH SCHEMA schema1;
26+
-- show all the extension functions and operators
27+
SELECT ne.nspname AS extschema, p.proname, np.nspname AS proschema
28+
FROM pg_catalog.pg_extension AS e
29+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
30+
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
31+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
32+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = p.pronamespace)
33+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
34+
ORDER BY 1, 3;
35+
extschema | proname | proschema
36+
-----------+------------------------------+-------------------
37+
schema1 | _create_ldb_operator_classes | _lantern_internal
38+
schema1 | ldb_generic_dist | schema1
39+
schema1 | ldb_generic_dist | schema1
40+
schema1 | hnsw_handler | schema1
41+
schema1 | cos_dist | schema1
42+
schema1 | hamming_dist | schema1
43+
schema1 | l2sq_dist | schema1
44+
(7 rows)
45+
46+
-- show all the extension operators
47+
SELECT ne.nspname AS extschema, op.oprname, np.nspname AS proschema
48+
FROM pg_catalog.pg_extension AS e
49+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
50+
INNER JOIN pg_catalog.pg_operator AS op ON (op.oid = d.objid)
51+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
52+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = op.oprnamespace)
53+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
54+
ORDER BY 1, 3;
55+
extschema | oprname | proschema
56+
-----------+---------+-----------
57+
schema1 | <-> | schema1
58+
schema1 | <-> | schema1
59+
(2 rows)
60+
61+
SET search_path TO public, schema1;
62+
-- extension function is accessible
63+
SELECT l2sq_dist(ARRAY[1.0, 2.0, 3.0], ARRAY[4.0, 5.0, 6.0]);
64+
l2sq_dist
65+
-----------
66+
27
67+
(1 row)
68+
69+
CREATE INDEX hnsw_index ON small_world USING hnsw(v) WITH (dim=3);
70+
INFO: done init usearch index
71+
INFO: inserted 8 elements
72+
INFO: done saving 8 vectors
73+
\set ON_ERROR_STOP off
74+
-- lantern does not support relocation.
75+
-- Postgres will not allow it to support this since its objects span over more than one schema
76+
ALTER EXTENSION lantern SET SCHEMA schema2;
77+
ERROR: extension "lantern" does not support SET SCHEMA
78+
\set ON_ERROR_STOP on
79+
SELECT ne.nspname AS extschema, op.oprname, np.nspname AS proschema
80+
FROM pg_catalog.pg_extension AS e
81+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
82+
INNER JOIN pg_catalog.pg_operator AS op ON (op.oid = d.objid)
83+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
84+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = op.oprnamespace)
85+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
86+
ORDER BY 1, 3;
87+
extschema | oprname | proschema
88+
-----------+---------+-----------
89+
schema1 | <-> | schema1
90+
schema1 | <-> | schema1
91+
(2 rows)
92+
93+
SET search_path TO public, schema2;
94+
--extension access method is still accessible since access methods are not schema-qualified
95+
CREATE INDEX hnsw_index2 ON small_world USING hnsw(v) WITH (dim=3);
96+
INFO: done init usearch index
97+
INFO: inserted 8 elements
98+
INFO: done saving 8 vectors
99+
\set ON_ERROR_STOP on
100+
-- extension function cannot be found without schema-qualification
101+
SELECT l2sq_dist(ARRAY[1.0, 2.0, 3.0], ARRAY[4.0, 5.0, 6.0]);
102+
ERROR: function l2sq_dist(numeric[], numeric[]) does not exist at character 8

‎test/schedule.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
# - 'test' lines may have multiple space-separated tests. All tests in a single 'test' line will be run in parallel
55

66
test_pgvector: hnsw_vector
7-
test: hnsw_config hnsw_correct hnsw_create hnsw_create_expr hnsw_dist_func hnsw_insert hnsw_select hnsw_todo hnsw_index_from_file hnsw_cost_estimate
7+
test: hnsw_config hnsw_correct hnsw_create hnsw_create_expr hnsw_dist_func hnsw_insert hnsw_select hnsw_todo hnsw_index_from_file hnsw_cost_estimate ext_relocation

‎test/sql/ext_relocation.sql

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
\ir utils/small_world_array.sql
2+
3+
DROP EXTENSION lantern;
4+
\set ON_ERROR_STOP off
5+
-- make sure the extension was dropped.
6+
CREATE INDEX ON small_world USING hnsw (v) WITH (dim=3);
7+
\set ON_ERROR_STOP on
8+
9+
-- test creating lantern on different schemas
10+
CREATE SCHEMA schema1;
11+
CREATE SCHEMA schema2;
12+
CREATE EXTENSION lantern WITH SCHEMA schema1;
13+
14+
-- show all the extension functions and operators
15+
SELECT ne.nspname AS extschema, p.proname, np.nspname AS proschema
16+
FROM pg_catalog.pg_extension AS e
17+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
18+
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
19+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
20+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = p.pronamespace)
21+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
22+
ORDER BY 1, 3;
23+
24+
-- show all the extension operators
25+
SELECT ne.nspname AS extschema, op.oprname, np.nspname AS proschema
26+
FROM pg_catalog.pg_extension AS e
27+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
28+
INNER JOIN pg_catalog.pg_operator AS op ON (op.oid = d.objid)
29+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
30+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = op.oprnamespace)
31+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
32+
ORDER BY 1, 3;
33+
34+
SET search_path TO public, schema1;
35+
36+
-- extension function is accessible
37+
SELECT l2sq_dist(ARRAY[1.0, 2.0, 3.0], ARRAY[4.0, 5.0, 6.0]);
38+
39+
CREATE INDEX hnsw_index ON small_world USING hnsw(v) WITH (dim=3);
40+
41+
\set ON_ERROR_STOP off
42+
-- lantern does not support relocation.
43+
-- Postgres will not allow it to support this since its objects span over more than one schema
44+
ALTER EXTENSION lantern SET SCHEMA schema2;
45+
\set ON_ERROR_STOP on
46+
47+
SELECT ne.nspname AS extschema, op.oprname, np.nspname AS proschema
48+
FROM pg_catalog.pg_extension AS e
49+
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
50+
INNER JOIN pg_catalog.pg_operator AS op ON (op.oid = d.objid)
51+
INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
52+
INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = op.oprnamespace)
53+
WHERE d.deptype = 'e' AND e.extname = 'lantern'
54+
ORDER BY 1, 3;
55+
56+
SET search_path TO public, schema2;
57+
--extension access method is still accessible since access methods are not schema-qualified
58+
CREATE INDEX hnsw_index2 ON small_world USING hnsw(v) WITH (dim=3);
59+
60+
\set ON_ERROR_STOP on
61+
-- extension function cannot be found without schema-qualification
62+
SELECT l2sq_dist(ARRAY[1.0, 2.0, 3.0], ARRAY[4.0, 5.0, 6.0]);
63+
\set ON_ERROR_STOP off
64+
SELECT schema1.l2sq_dist(ARRAY[1.0, 2.0, 3.0], ARRAY[4.0, 5.0, 6.0]);

0 commit comments

Comments
 (0)