From 1ec1b32aa37ce9f1be6ca3db1ea7cb26b7e1c753 Mon Sep 17 00:00:00 2001 From: Vishwaraj Anand Date: Thu, 6 Mar 2025 16:29:49 +0000 Subject: [PATCH 1/5] chore: add migration guides tests --- samples.cloudbuild.yaml | 112 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 samples.cloudbuild.yaml diff --git a/samples.cloudbuild.yaml b/samples.cloudbuild.yaml new file mode 100644 index 00000000..519b9361 --- /dev/null +++ b/samples.cloudbuild.yaml @@ -0,0 +1,112 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: Install dependencies + name: python:${_VERSION} + entrypoint: pip + args: ["install", "--user", "-r", "samples/migrations/requirements.txt"] + + - id: Install module (and test requirements) + name: python:${_VERSION} + entrypoint: pip + args: ["install", ".[test]", "--user"] + + - id: proxy-install + name: alpine:3.10 + entrypoint: sh + args: + - -c + - | + wget -O /workspace/alloydb-auth-proxy https://storage.googleapis.com/alloydb-auth-proxy/v1.11.0/alloydb-auth-proxy.linux.386 + chmod +x /workspace/alloydb-auth-proxy + + - id: Run samples tests + name: python:${_VERSION} + entrypoint: /bin/bash + args: + - "-c" + - | + /workspace/alloydb-auth-proxy --port ${_DATABASE_PORT} ${_INSTANCE_CONNECTION_NAME} & sleep 2; + python -m pytest samples/migrations/ + env: + - "PROJECT_ID=$PROJECT_ID" + - "INSTANCE_ID=$_INSTANCE_ID" + - "CLUSTER_ID=$_CLUSTER_ID" + - "DATABASE_ID=$_DATABASE_ID" + - "REGION=$_REGION" + - "OMNI_DATABASE_ID=$_OMNI_DATABASE_ID" + - "IP_ADDRESS=$_IP_ADDRESS" + + secretEnv: + [ + "DB_USER", + "DB_PASSWORD", + "OMNI_HOST", + "OMNI_USER", + "OMNI_PASSWORD", + "IAM_ACCOUNT", + "PINECONE_API_KEY", + "PINECONE_INDEX_NAME", + "CHROMADB_COLLECTION_NAME", + "MILVUS_COLLECTION_NAME", + "WEAVIATE_API_KEY", + "WEAVIATE_URL", + "WEAVIATE_COLLECTION_NAME", + "QDRANT_COLLECTION_NAME", + ] + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/DB_USER/versions/1 + env: "DB_USER" + - versionName: projects/$PROJECT_ID/secrets/DB_PASSWORD/versions/1 + env: "DB_PASSWORD" + - versionName: projects/$PROJECT_ID/secrets/OMNI_HOST/versions/1 + env: "OMNI_HOST" + - versionName: projects/$PROJECT_ID/secrets/OMNI_USER/versions/1 + env: "OMNI_USER" + - versionName: projects/$PROJECT_ID/secrets/OMNI_PASSWORD/versions/1 + env: "OMNI_PASSWORD" + - versionName: projects/$PROJECT_ID/secrets/service_account_email/versions/1 + env: "IAM_ACCOUNT" + - versionName: projects/$PROJECT_ID/secrets/PINECONE_API_KEY/versions/1 + env: "PINECONE_API_KEY" + - versionName: projects/$PROJECT_ID/secrets/PINECONE_INDEX_NAME/versions/1 + env: "PINECONE_INDEX_NAME" + - versionName: projects/$PROJECT_ID/secrets/CHROMADB_COLLECTION_NAME/versions/1 + env: "CHROMADB_COLLECTION_NAME" + - versionName: projects/$PROJECT_ID/secrets/MILVUS_COLLECTION_NAME/versions/1 + env: "MILVUS_COLLECTION_NAME" + # TODO: undo once weaviate cloud is available + # - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_API_KEY/versions/1 + # env: "WEAVIATE_API_KEY" + # - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_URL/versions/1 + # env: "WEAVIATE_URL" + - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_COLLECTION_NAME/versions/1 + env: "WEAVIATE_COLLECTION_NAME" + - versionName: projects/$PROJECT_ID/secrets/QDRANT_COLLECTION_NAME/versions/1 + env: "QDRANT_COLLECTION_NAME" + + +substitutions: + _VERSION: "3.10" + _INSTANCE_CONNECTION_NAME: projects/${PROJECT_ID}/locations/${_REGION}/clusters/${_CLUSTER_ID}/instances/${_INSTANCE_ID} + _DATABASE_PORT: "5432" + _IP_ADDRESS: "127.0.0.1" # required in other tests else pytest fails + +options: + pool: + name: "projects/langchain-alloydb-testing/locations/us-central1/workerPools/my-pool2" + dynamicSubstitutions: true From 03821e1ed197dd04fe7597687fd57ed404399391 Mon Sep 17 00:00:00 2001 From: Vishwaraj Anand Date: Thu, 6 Mar 2025 18:18:05 +0000 Subject: [PATCH 2/5] fix: remove weaviate keys --- samples.cloudbuild.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/samples.cloudbuild.yaml b/samples.cloudbuild.yaml index 519b9361..5ef0f926 100644 --- a/samples.cloudbuild.yaml +++ b/samples.cloudbuild.yaml @@ -61,8 +61,6 @@ steps: "PINECONE_INDEX_NAME", "CHROMADB_COLLECTION_NAME", "MILVUS_COLLECTION_NAME", - "WEAVIATE_API_KEY", - "WEAVIATE_URL", "WEAVIATE_COLLECTION_NAME", "QDRANT_COLLECTION_NAME", ] @@ -89,11 +87,6 @@ availableSecrets: env: "CHROMADB_COLLECTION_NAME" - versionName: projects/$PROJECT_ID/secrets/MILVUS_COLLECTION_NAME/versions/1 env: "MILVUS_COLLECTION_NAME" - # TODO: undo once weaviate cloud is available - # - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_API_KEY/versions/1 - # env: "WEAVIATE_API_KEY" - # - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_URL/versions/1 - # env: "WEAVIATE_URL" - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_COLLECTION_NAME/versions/1 env: "WEAVIATE_COLLECTION_NAME" - versionName: projects/$PROJECT_ID/secrets/QDRANT_COLLECTION_NAME/versions/1 From 8db86abb23203199dee3c96431cee94621005ac9 Mon Sep 17 00:00:00 2001 From: Vishwaraj Anand Date: Thu, 6 Mar 2025 18:34:29 +0000 Subject: [PATCH 3/5] fix: add weaviate cloud tests --- samples.cloudbuild.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/samples.cloudbuild.yaml b/samples.cloudbuild.yaml index 5ef0f926..91aa5e75 100644 --- a/samples.cloudbuild.yaml +++ b/samples.cloudbuild.yaml @@ -61,6 +61,8 @@ steps: "PINECONE_INDEX_NAME", "CHROMADB_COLLECTION_NAME", "MILVUS_COLLECTION_NAME", + "WEAVIATE_API_KEY", + "WEAVIATE_URL", "WEAVIATE_COLLECTION_NAME", "QDRANT_COLLECTION_NAME", ] @@ -87,6 +89,10 @@ availableSecrets: env: "CHROMADB_COLLECTION_NAME" - versionName: projects/$PROJECT_ID/secrets/MILVUS_COLLECTION_NAME/versions/1 env: "MILVUS_COLLECTION_NAME" + - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_API_KEY/versions/1 + env: "WEAVIATE_API_KEY" + - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_URL/versions/1 + env: "WEAVIATE_URL" - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_COLLECTION_NAME/versions/1 env: "WEAVIATE_COLLECTION_NAME" - versionName: projects/$PROJECT_ID/secrets/QDRANT_COLLECTION_NAME/versions/1 From 76b78cf285329a1ef29923854907a389fdb64032 Mon Sep 17 00:00:00 2001 From: Vishwaraj Anand Date: Thu, 6 Mar 2025 19:18:47 +0000 Subject: [PATCH 4/5] fix: pinecone cloud spec --- samples/migrations/test_pinecone_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/migrations/test_pinecone_migration.py b/samples/migrations/test_pinecone_migration.py index c3cb607c..1251c5b9 100644 --- a/samples/migrations/test_pinecone_migration.py +++ b/samples/migrations/test_pinecone_migration.py @@ -76,7 +76,7 @@ def create_pinecone_index( name=pinecone_index_name, dimension=768, metric="cosine", - spec=ServerlessSpec(cloud="aws", region="us-east-1"), + spec=ServerlessSpec(cloud="gcp", region="us-central1"), ) while not client.describe_index(pinecone_index_name).status["ready"]: time.sleep(1) From 8d4a459ee5168a58d409f711d903ae62fb328404 Mon Sep 17 00:00:00 2001 From: Vishwaraj Anand Date: Mon, 10 Mar 2025 18:56:49 +0000 Subject: [PATCH 5/5] chore: pr comments --- samples.cloudbuild.yaml | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/samples.cloudbuild.yaml b/samples.cloudbuild.yaml index 91aa5e75..c2c556c2 100644 --- a/samples.cloudbuild.yaml +++ b/samples.cloudbuild.yaml @@ -23,22 +23,12 @@ steps: entrypoint: pip args: ["install", ".[test]", "--user"] - - id: proxy-install - name: alpine:3.10 - entrypoint: sh - args: - - -c - - | - wget -O /workspace/alloydb-auth-proxy https://storage.googleapis.com/alloydb-auth-proxy/v1.11.0/alloydb-auth-proxy.linux.386 - chmod +x /workspace/alloydb-auth-proxy - - id: Run samples tests name: python:${_VERSION} entrypoint: /bin/bash args: - "-c" - | - /workspace/alloydb-auth-proxy --port ${_DATABASE_PORT} ${_INSTANCE_CONNECTION_NAME} & sleep 2; python -m pytest samples/migrations/ env: - "PROJECT_ID=$PROJECT_ID" @@ -48,6 +38,11 @@ steps: - "REGION=$_REGION" - "OMNI_DATABASE_ID=$_OMNI_DATABASE_ID" - "IP_ADDRESS=$_IP_ADDRESS" + - "PINECONE_INDEX_NAME=$_PINECONE_INDEX_NAME" + - "WEAVIATE_COLLECTION_NAME=$_WEAVIATE_COLLECTION_NAME" + - "CHROMADB_COLLECTION_NAME=$_CHROMADB_COLLECTION_NAME" + - "MILVUS_COLLECTION_NAME=$_MILVUS_COLLECTION_NAME" + - "QDRANT_COLLECTION_NAME=$_QDRANT_COLLECTION_NAME" secretEnv: [ @@ -58,13 +53,8 @@ steps: "OMNI_PASSWORD", "IAM_ACCOUNT", "PINECONE_API_KEY", - "PINECONE_INDEX_NAME", - "CHROMADB_COLLECTION_NAME", - "MILVUS_COLLECTION_NAME", "WEAVIATE_API_KEY", "WEAVIATE_URL", - "WEAVIATE_COLLECTION_NAME", - "QDRANT_COLLECTION_NAME", ] availableSecrets: @@ -83,29 +73,21 @@ availableSecrets: env: "IAM_ACCOUNT" - versionName: projects/$PROJECT_ID/secrets/PINECONE_API_KEY/versions/1 env: "PINECONE_API_KEY" - - versionName: projects/$PROJECT_ID/secrets/PINECONE_INDEX_NAME/versions/1 - env: "PINECONE_INDEX_NAME" - - versionName: projects/$PROJECT_ID/secrets/CHROMADB_COLLECTION_NAME/versions/1 - env: "CHROMADB_COLLECTION_NAME" - - versionName: projects/$PROJECT_ID/secrets/MILVUS_COLLECTION_NAME/versions/1 - env: "MILVUS_COLLECTION_NAME" - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_API_KEY/versions/1 env: "WEAVIATE_API_KEY" - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_URL/versions/1 env: "WEAVIATE_URL" - - versionName: projects/$PROJECT_ID/secrets/WEAVIATE_COLLECTION_NAME/versions/1 - env: "WEAVIATE_COLLECTION_NAME" - - versionName: projects/$PROJECT_ID/secrets/QDRANT_COLLECTION_NAME/versions/1 - env: "QDRANT_COLLECTION_NAME" - substitutions: _VERSION: "3.10" _INSTANCE_CONNECTION_NAME: projects/${PROJECT_ID}/locations/${_REGION}/clusters/${_CLUSTER_ID}/instances/${_INSTANCE_ID} _DATABASE_PORT: "5432" _IP_ADDRESS: "127.0.0.1" # required in other tests else pytest fails + _PINECONE_INDEX_NAME: "langchain-test-index" + _WEAVIATE_COLLECTION_NAME: "example_collection" + _CHROMADB_COLLECTION_NAME: "example_collection" + _MILVUS_COLLECTION_NAME: "langchain_example" + _QDRANT_COLLECTION_NAME: "demo_collection" options: - pool: - name: "projects/langchain-alloydb-testing/locations/us-central1/workerPools/my-pool2" dynamicSubstitutions: true