From 4963b7c93e0586f5e5cec22b4b1175db02dfd811 Mon Sep 17 00:00:00 2001 From: rahulkumar Date: Mon, 28 Oct 2024 14:56:29 -0700 Subject: [PATCH 1/5] avs python 3.0.0 changes supporting avs 0.11.1 --- .internal/docker-compose-prism-dev.yml | 2 +- .internal/docker-compose-quote-search-dev.yml | 2 +- basic-search/README.md | 2 +- basic-search/requirements.txt | 2 +- docker/README.md | 2 +- docker/docker-compose.yaml | 2 +- kubernetes/manifests/avs-gke-values.yaml | 2 +- prism-image-search/README.md | 10 +++++----- prism-image-search/docker-compose.yml | 2 +- prism-image-search/prism/requirements.txt | 4 ++-- quote-semantic-search/README.md | 2 +- quote-semantic-search/docker-compose.yml | 2 +- quote-semantic-search/quote-search/requirements.txt | 4 ++-- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.internal/docker-compose-prism-dev.yml b/.internal/docker-compose-prism-dev.yml index 925bc9c..d89644e 100644 --- a/.internal/docker-compose-prism-dev.yml +++ b/.internal/docker-compose-prism-dev.yml @@ -11,7 +11,7 @@ services: - "--config-file" - "/opt/aerospike/etc/aerospike/aerospike.conf" avs: - image: aerospike/aerospike-vector-search:0.10.0 + image: aerospike/aerospike-vector-search:0.11.1 # ports: # - "5000:5000" networks: diff --git a/.internal/docker-compose-quote-search-dev.yml b/.internal/docker-compose-quote-search-dev.yml index 46c444f..52eee63 100644 --- a/.internal/docker-compose-quote-search-dev.yml +++ b/.internal/docker-compose-quote-search-dev.yml @@ -11,7 +11,7 @@ services: - "--config-file" - "/opt/aerospike/etc/aerospike/aerospike.conf" avs: - image: aerospike/aerospike-vector-search:0.10.0 + image: aerospike/aerospike-vector-search:0.11.1 # ports: # - "5002:5002" networks: diff --git a/basic-search/README.md b/basic-search/README.md index 5cc68f3..ddc1f45 100644 --- a/basic-search/README.md +++ b/basic-search/README.md @@ -6,7 +6,7 @@ vector record insertion, and basic ANN query against the AVS server using the Py ## Prerequisites 1. A Python 3.10 - 3.11 environment and familiarity with the Python programming language (see [Setup Python Virtual Environment](../prism-image-search/README.md#setup-python-virtual-environment)). -1. An Aerospike Vector Search host (sandbox or local) running AVS 0.10.0 or newer. +2. An Aerospike Vector Search host (sandbox or local) running AVS 0.11.0 or 0.11.1. ## Setup build Python Virtual Environment diff --git a/basic-search/requirements.txt b/basic-search/requirements.txt index 9ac9af2..ac00148 100644 --- a/basic-search/requirements.txt +++ b/basic-search/requirements.txt @@ -1 +1 @@ -aerospike-vector-search==2.0.0 \ No newline at end of file +aerospike-vector-search==3.0.0 \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 101c5fc..a5c136e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -33,7 +33,7 @@ docker run -d \ -p 5000:5000 \ -p 5040:5040 \ -v ./config:/etc/aerospike-vector-search \ -aerospike/aerospike-vector-search:0.10.0 +aerospike/aerospike-vector-search:0.11.1 ``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 31a9448..82218df 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -20,7 +20,7 @@ services: networks: - svc aerospike-vector-search: - image: aerospike/aerospike-vector-search:0.10.0 + image: aerospike/aerospike-vector-search:0.11.1 container_name: "aerospike-vector-search" depends_on: aerospike-cluster: diff --git a/kubernetes/manifests/avs-gke-values.yaml b/kubernetes/manifests/avs-gke-values.yaml index 414db92..7ab97ec 100644 --- a/kubernetes/manifests/avs-gke-values.yaml +++ b/kubernetes/manifests/avs-gke-values.yaml @@ -47,7 +47,7 @@ image: repository: "aerospike/aerospike-vector-search" pullPolicy: "IfNotPresent" # Overrides the image tag whose default is the chart appVersion. - tag: "0.10.0" + tag: "0.11.1" extraSecretVolumeMounts: - name: aerospike-tls mountPath: "/etc/ssl/certs" diff --git a/prism-image-search/README.md b/prism-image-search/README.md index e7037e7..4bafbbb 100644 --- a/prism-image-search/README.md +++ b/prism-image-search/README.md @@ -10,7 +10,7 @@ Aerospike Vector Search (AVS) performs Approximate Nearest Neighbor(ANN) search You don't have to know Aerospike to get started, but you do need the following: 1. A Python 3.10 - 3.11 environment and familiarity with the Python programming language (see [Setup Python Virtual Environment](./README.md#setup-python-virtual-environment)). -1. An Aerospike Vector Search host (preview environment or local) running AVS 0.10.0 or newer. +2. An Aerospike Vector Search host (preview environment or local) running AVS 0.11.1 or newer. ## Configure AVS host @@ -53,8 +53,8 @@ description. ## Install using docker compose -If you have a license key, you can easily setup Aerospike, AVS, and the prism-image-search -app using docker-compose. When using docker-compose, you'll need to place your images in `container-volumes` +If you have a license key, you can easily set up Aerospike, AVS, and the prism-image-search +app using `docker-compose`. When using `docker-compose`, you'll need to place your images in `container-volumes/prism/images/static/data` ```shell ln -s ~/Pictures container-volumes/prism/images/static/data @@ -68,11 +68,11 @@ docker build -t prism . -f Dockerfile-prism ### 2. Add features.conf AVS needs an Aerospike features.conf file with the vector-search feature enabled. -Add your features.conf file to container-volumes/avs/etc/avs. +Add your features.conf file to `container-volumes/avs/etc/aerospike-vector-search`. ### 3. Start the environment ``` -docker compose up +docker-compose up ``` ## Developing diff --git a/prism-image-search/docker-compose.yml b/prism-image-search/docker-compose.yml index 8ebbe1e..81aa4f1 100644 --- a/prism-image-search/docker-compose.yml +++ b/prism-image-search/docker-compose.yml @@ -19,7 +19,7 @@ services: timeout: 20s retries: 10 avs: - image: aerospike/aerospike-vector-search:0.10.0 + image: aerospike/aerospike-vector-search:0.11.1 depends_on: aerospike: condition: service_healthy diff --git a/prism-image-search/prism/requirements.txt b/prism-image-search/prism/requirements.txt index dd932a4..64e027e 100644 --- a/prism-image-search/prism/requirements.txt +++ b/prism-image-search/prism/requirements.txt @@ -1,6 +1,6 @@ # TODO: Include exact versions # Aerospike dependencies -aerospike-vector-search==2.0.0 +aerospike-vector-search==3.0.0 # Flask framework flask~=2.3.2 @@ -11,7 +11,7 @@ waitress # Image processing numpy==1.26.4 # fix issues in dependencies after 2.0.0 release -sentence-transformers==2.2.2 +sentence-transformers==3.2.1 torch==2.0.1 tqdm~=4.66.1 Pillow~=10.0.0 diff --git a/quote-semantic-search/README.md b/quote-semantic-search/README.md index 8288ee7..086df89 100644 --- a/quote-semantic-search/README.md +++ b/quote-semantic-search/README.md @@ -11,7 +11,7 @@ Aerospike Vector Search (AVS) performs Approximate Nearest Neighbor(ANN) search You don't have to know Aerospike to get started, but you do need the following: 1. A Python 3.10 - 3.11 environment and familiarity with the Python programming language (see [Setup Python Virtual Environment](../prism-image-search/README.md#setup-python-virtual-environment)). -1. An Aerospike Vector Search host (preview or local) running AVS 0.10.0 or newer. +2. An Aerospike Vector Search host (preview or local) running AVS 0.11.1. ## Configure AVS host diff --git a/quote-semantic-search/docker-compose.yml b/quote-semantic-search/docker-compose.yml index acdfed6..c90c4a7 100644 --- a/quote-semantic-search/docker-compose.yml +++ b/quote-semantic-search/docker-compose.yml @@ -22,7 +22,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.10.0 + image: aerospike/aerospike-vector-search:0.11.1 # ports: # - "5002:5002" networks: diff --git a/quote-semantic-search/quote-search/requirements.txt b/quote-semantic-search/quote-search/requirements.txt index f961944..6529f3a 100644 --- a/quote-semantic-search/quote-search/requirements.txt +++ b/quote-semantic-search/quote-search/requirements.txt @@ -1,6 +1,6 @@ # TODO: Include exact versions # Aerospike dependencies -aerospike-vector-search==2.0.0 +aerospike-vector-search==3.0.0 # Flask framework @@ -12,7 +12,7 @@ waitress # Quote processing numpy==1.26.4 # fix issues in dependencies after 2.0.0 release -sentence-transformers==2.2.2 +sentence-transformers==3.2.1 torch==2.0.1 tqdm~=4.66.1 From 6ca7b4c6469392c7880312370ffc754ed737c2ab Mon Sep 17 00:00:00 2001 From: rahulkumar Date: Mon, 28 Oct 2024 15:19:28 -0700 Subject: [PATCH 2/5] avs python 3.0.0 changes supporting avs 0.11.1 --- basic-search/README.md | 2 +- kubernetes/manifests/avs-gke-values-auth.yaml | 2 +- prism-image-search/README.md | 6 +- .../prism/images/static/data/avs.png | Bin 0 -> 1840 bytes prism-image-search/docker-compose-dev.yml | 53 +++++++++++++++++ prism-image-search/docker-compose.yml | 2 +- quote-semantic-search/docker-compose-dev.yml | 55 ++++++++++++++++++ quote-semantic-search/docker-compose.yml | 2 +- 8 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 prism-image-search/container-volumes/prism/images/static/data/avs.png create mode 100644 prism-image-search/docker-compose-dev.yml create mode 100644 quote-semantic-search/docker-compose-dev.yml diff --git a/basic-search/README.md b/basic-search/README.md index ddc1f45..fa7c052 100644 --- a/basic-search/README.md +++ b/basic-search/README.md @@ -6,7 +6,7 @@ vector record insertion, and basic ANN query against the AVS server using the Py ## Prerequisites 1. A Python 3.10 - 3.11 environment and familiarity with the Python programming language (see [Setup Python Virtual Environment](../prism-image-search/README.md#setup-python-virtual-environment)). -2. An Aerospike Vector Search host (sandbox or local) running AVS 0.11.0 or 0.11.1. +2. An Aerospike Vector Search host (sandbox or local) running AVS 0.11.1 or newer. ## Setup build Python Virtual Environment diff --git a/kubernetes/manifests/avs-gke-values-auth.yaml b/kubernetes/manifests/avs-gke-values-auth.yaml index 15f4eb7..e9799db 100644 --- a/kubernetes/manifests/avs-gke-values-auth.yaml +++ b/kubernetes/manifests/avs-gke-values-auth.yaml @@ -91,7 +91,7 @@ image: repository: "aerospike/aerospike-vector-search" pullPolicy: "IfNotPresent" # Overrides the image tag whose default is the chart appVersion. - tag: "0.10.0" + tag: "0.11.1" extraSecretVolumeMounts: - name: aerospike-tls mountPath: "/etc/ssl/certs" diff --git a/prism-image-search/README.md b/prism-image-search/README.md index 4bafbbb..e442a0c 100644 --- a/prism-image-search/README.md +++ b/prism-image-search/README.md @@ -53,8 +53,8 @@ description. ## Install using docker compose -If you have a license key, you can easily set up Aerospike, AVS, and the prism-image-search -app using `docker-compose`. When using `docker-compose`, you'll need to place your images in `container-volumes/prism/images/static/data` +If you have a license key, you can easily setup Aerospike, AVS, and the prism-image-search +app using docker-compose. When using docker-compose, you'll need to place your images in `container-volumes/prism/images/static/data` ```shell ln -s ~/Pictures container-volumes/prism/images/static/data @@ -72,7 +72,7 @@ Add your features.conf file to `container-volumes/avs/etc/aerospike-vector-searc ### 3. Start the environment ``` -docker-compose up +docker compose up ``` ## Developing diff --git a/prism-image-search/container-volumes/prism/images/static/data/avs.png b/prism-image-search/container-volumes/prism/images/static/data/avs.png new file mode 100644 index 0000000000000000000000000000000000000000..c70a759ae5d96a4c30652938b618ce00236f6ddd GIT binary patch literal 1840 zcmV-02haG4P)k7RA_qopAN-6gce zmiC#uxwF0wGrKd>owr~)$t0bf_ukxd@4NTix$l*6&S5!X#j<4mlz0`OYI;P~^!{Z< z97zK}8UT)|nm$ks8Ke^zNUx`#I;%lmDsc`YqX2M2{4p5|E3(PUB%Vdm$yj*U>LHOoD@&MKSuaY) z!es+jG8T@PYf~vK+gx3m#qF@Ue$`8~%o&KvcyHB-*D_cK0DEP>SL#$}j71n@{onK` zKN!C~n7ZE}Qw9UR6K9MC7-I@!45RrFewZvQ%&#fYegGBZw_LsvXN*OW1OQ9|z&QZe z<*+MeZ$qdkQdqzb`6^1BF&1QuB^YD4W@|Pn=*6wgkyFM^-I}%cY=+xK3Y(-^P|=`Q z5#nT-6ab6@z$XB(-PB|Z0D1u+#5r%nNWZhDMDo=TK z3+Ft-IUjIB8u9n^>c8moXiK<;JkO zPF}hmIeF=Nzn8uwmqMwom29c9{T;j@YBRWap#X|k5-!L_$e(j%%+Ji5V>Ev{<1LK)f_s?H=TNt)0` zRTeBa4dKZNQnqJ!Ca!9FRMqq}krj`3q)T-3ij=7zR)eytrhDF9bxmB=^dOy8(M)?@ zn0@s+*$#FKn%Xf_)`!YIqiQ-Tz_Qd-YF0c#oQS2!uZpN!+W@VdT|l|Cx&DUfW-u8G z`!UT6NXEh#J0g_!inTT@IctZ$zIZUuxYgV(Xs4LsAzso>f_y2TjD?HZdinMJ{EfpO z!}=}HFX~*w*st@PA8{LO7-MmY2?$JOH@N9|MHnaz|3%aOw?F-A(ctS=*Us*L;h9w}jSW9Lt&&y}$ygXykNAzd(|={g?tGrj z%@jv(?K}dnzWd=(RnxDL+ofrSqFX9P@D(kd5_Url1Hf+cl&;|m&>Gqdd$w(Wy4s?b zgea;z#!@X#2+(`ikLIRf@b?MG|1)oHgH}f4B#rxIEL@oOqQyl&_zVpzj?mKB0Q*9l zAW*l`Jb2u6d`lMM#%dA@)I-mn7va+A9k@Jp8|FAOMA&Nzz{S4{W+Uepm!u-;XVa=j_D9#5)#E z$Ef5}b{+TP<0Uo^5t+WezALQ-*)x( z{ky+SRZspqWo1;*xQ$(ldZgR-|nX8f0i2I@;acee`?%)+??7H3Mh5?Mpz1Q4+?O+W%NW4w@|=63&NgGR(QRt^1r2YuI2=ta5&C6zam|RY7l=M eusqAl0Pruhmd0XHj`c180000 Date: Mon, 28 Oct 2024 15:27:17 -0700 Subject: [PATCH 3/5] avs python 3.0.0 changes supporting avs 0.11.1 --- quote-semantic-search/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quote-semantic-search/README.md b/quote-semantic-search/README.md index 086df89..aa8b76c 100644 --- a/quote-semantic-search/README.md +++ b/quote-semantic-search/README.md @@ -62,7 +62,7 @@ Add your features.conf file to container-volumes/avs/etc/aerospike-vector-search ### 3. Start the environment ``` -docker compose up +docker compose -f docker-compose-dev.yml up ``` ## Developing From 68e97a3879474714289aaccc301358e5330ca784 Mon Sep 17 00:00:00 2001 From: David Gerchikov Date: Fri, 1 Nov 2024 02:36:06 +0200 Subject: [PATCH 4/5] Update full-create-and-install.sh (#67) :+1: --- kubernetes/full-create-and-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/full-create-and-install.sh b/kubernetes/full-create-and-install.sh index 5312eb4..6db50e4 100755 --- a/kubernetes/full-create-and-install.sh +++ b/kubernetes/full-create-and-install.sh @@ -417,7 +417,7 @@ deploy_avs_helm_chart() { helm repo add aerospike-helm https://artifact.aerospike.io/artifactory/api/helm/aerospike-helm helm repo update if [ -z "$CHART_LOCATION" ]; then - helm install avs-gke --values $BUILD_DIR/manifests/avs-gke-values.yaml --namespace avs aerospike-helm/aerospike-vector-search --version 0.4.1 --wait + helm install avs-gke --values $BUILD_DIR/manifests/avs-gke-values.yaml --namespace avs aerospike-helm/aerospike-vector-search --version 0.6.0 --wait else helm install avs-gke --values $BUILD_DIR/manifests/avs-gke-values.yaml --namespace avs "$CHART_LOCATION" --wait fi From 39621f300b689ff22d034e62e44e34c64f5cd1a1 Mon Sep 17 00:00:00 2001 From: "Joe M." Date: Fri, 1 Nov 2024 15:01:02 -0700 Subject: [PATCH 5/5] VEC-424 Update Aerospike storage paths and sizes (#69) --- kubernetes/manifests/aerospike-cr-auth.yaml | 36 ++++++++++++++++++--- kubernetes/manifests/aerospike-cr.yaml | 36 ++++++++++++++++++--- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/kubernetes/manifests/aerospike-cr-auth.yaml b/kubernetes/manifests/aerospike-cr-auth.yaml index 003ccff..cf52c18 100644 --- a/kubernetes/manifests/aerospike-cr-auth.yaml +++ b/kubernetes/manifests/aerospike-cr-auth.yaml @@ -24,13 +24,28 @@ spec: size: 1Gi - name: avs-meta aerospike: - path: /avs/dev/xvdf + path: /avs-meta/dev/xvdf + source: + persistentVolume: + storageClass: ssd + volumeMode: Block + size: 2Gi + - name: avs-index + aerospike: + path: /avs-index/dev/xvdf + source: + persistentVolume: + storageClass: ssd + volumeMode: Block + size: 20Gi + - name: avs-data + aerospike: + path: /avs-data/dev/xvdf source: persistentVolume: storageClass: ssd volumeMode: Block size: 20Gi - - name: ns aerospike: path: /test/dev/xvdf @@ -38,7 +53,7 @@ spec: persistentVolume: storageClass: ssd volumeMode: Block - size: 20Gi + size: 1Gi - name: aerospike-config-secret source: secret: @@ -128,7 +143,18 @@ spec: type: device devices: - /test/dev/xvdf - + - name: avs-index + replication-factor: 2 + storage-engine: + type: device + devices: + - /avs-index/dev/xvdf + - name: avs-data + replication-factor: 2 + storage-engine: + type: device + devices: + - /avs-data/dev/xvdf - name: avs-meta nsup-period: 600 nsup-threads: 2 @@ -137,4 +163,4 @@ spec: storage-engine: type: device devices: - - /avs/dev/xvdf \ No newline at end of file + - /avs-meta/dev/xvdf \ No newline at end of file diff --git a/kubernetes/manifests/aerospike-cr.yaml b/kubernetes/manifests/aerospike-cr.yaml index 5ad7c84..29f1355 100644 --- a/kubernetes/manifests/aerospike-cr.yaml +++ b/kubernetes/manifests/aerospike-cr.yaml @@ -24,13 +24,28 @@ spec: size: 1Gi - name: avs-meta aerospike: - path: /avs/dev/xvdf + path: /avs-meta/dev/xvdf + source: + persistentVolume: + storageClass: ssd + volumeMode: Block + size: 2Gi + - name: avs-index + aerospike: + path: /avs-index/dev/xvdf + source: + persistentVolume: + storageClass: ssd + volumeMode: Block + size: 20Gi + - name: avs-data + aerospike: + path: /avs-data/dev/xvdf source: persistentVolume: storageClass: ssd volumeMode: Block size: 20Gi - - name: ns aerospike: path: /test/dev/xvdf @@ -38,7 +53,7 @@ spec: persistentVolume: storageClass: ssd volumeMode: Block - size: 20Gi + size: 1Gi - name: aerospike-config-secret source: secret: @@ -90,7 +105,18 @@ spec: type: device devices: - /test/dev/xvdf - + - name: avs-index + replication-factor: 2 + storage-engine: + type: device + devices: + - /avs-index/dev/xvdf + - name: avs-data + replication-factor: 2 + storage-engine: + type: device + devices: + - /avs-data/dev/xvdf - name: avs-meta nsup-period: 600 nsup-threads: 2 @@ -99,4 +125,4 @@ spec: storage-engine: type: device devices: - - /avs/dev/xvdf \ No newline at end of file + - /avs-meta/dev/xvdf \ No newline at end of file