From 1048b6bb14a1aa77ae794a2d7af9ffc9faf7d555 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Mon, 20 Jan 2025 11:35:16 +0400 Subject: [PATCH 01/10] feat: start postgres module with integration tests --- rig-postgres/Cargo.toml | 29 ++++++++++++ rig-postgres/tests/integration_tests.rs | 51 +++++++++++++++++++++ rig-postgres/tests/migrations/001_setup.sql | 13 ++++++ 3 files changed, 93 insertions(+) create mode 100644 rig-postgres/Cargo.toml create mode 100644 rig-postgres/tests/integration_tests.rs create mode 100644 rig-postgres/tests/migrations/001_setup.sql diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml new file mode 100644 index 0000000000..e57d793bba --- /dev/null +++ b/rig-postgres/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "rig-postgres" +version = "0.1.0" +edition = "2021" +description = "PostgreSQL-based vector store implementation for the rig framework" +license = "MIT" + +[dependencies] +pgvector = { version = "0.4.0", features = ["postgres"] } +rig-core = { path = "../rig-core", version = "0.6.0", features = ["derive"] } +serde = { version = "1.0.215", features = ["derive"] } +serde_json = "1.0.133" +tokio-postgres = "0.7.12" +tracing = "0.1.40" +sqlx = { version = "0.8", features = [ + "runtime-tokio", + "postgres", + "uuid", + "json", +] } + +[dev-dependencies] +anyhow = "1.0.94" +log = "0.4.22" +tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread"] } +tokio-test = "0.4.4" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +uuid = { version = "1.11.0", features = ["v4"] } +testcontainers = "0.23.1" diff --git a/rig-postgres/tests/integration_tests.rs b/rig-postgres/tests/integration_tests.rs new file mode 100644 index 0000000000..03b5fb7751 --- /dev/null +++ b/rig-postgres/tests/integration_tests.rs @@ -0,0 +1,51 @@ +use sqlx::postgres::PgPoolOptions; +use testcontainers::{ + core::{IntoContainerPort, WaitFor}, + runners::AsyncRunner, + GenericImage, ImageExt, +}; + +const POSTGRES_PORT: u16 = 5432; + +#[tokio::test] +async fn vector_search_test() { + // Setup a local postgres container for testing. NOTE: docker service must be running. + let container = GenericImage::new("pgvector/pgvector", "pg17") + .with_wait_for(WaitFor::message_on_stderr( + "database system is ready to accept connections", + )) + .with_exposed_port(POSTGRES_PORT.tcp()) + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_DB", "rig") + .start() + .await + .expect("Failed to start postgres with pgvector container"); + + let host = container.get_host().await.unwrap().to_string(); + let port = container + .get_host_port_ipv4(POSTGRES_PORT) + .await + .expect("Error getting docker port"); + + println!("Container started on host:port {}:{}", host, port); + + // connect to Postgres + let pg_pool = PgPoolOptions::new() + .max_connections(50) + .idle_timeout(std::time::Duration::from_secs(5)) + .connect(&format!( + "postgres://postgres:postgres@{}:{}/rig", + host, port + )) + .await + .expect("Failed to create postgres pool"); + + // run migrations on Postgres + sqlx::migrate!("./tests/migrations") + .run(&pg_pool) + .await + .expect("Failed to run migrations"); + + println!("Connected to postgres"); +} diff --git a/rig-postgres/tests/migrations/001_setup.sql b/rig-postgres/tests/migrations/001_setup.sql new file mode 100644 index 0000000000..5867397e90 --- /dev/null +++ b/rig-postgres/tests/migrations/001_setup.sql @@ -0,0 +1,13 @@ +-- ensure extension is installed +CREATE EXTENSION IF NOT EXISTS vector; + +-- create table with embeddings using 1536 dimensions (text-embedding-3-small) +CREATE TABLE documents ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + document jsonb NOT NULL, + embeddings vector(1536) +); + +-- create index on embeddings +CREATE INDEX IF NOT EXISTS document_embeddings_idx ON documents +USING hnsw(embeddings vector_cosine_ops); From 278d8c719858347fa9031c6c4c56fe635db25c8f Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Mon, 20 Jan 2025 18:28:14 +0400 Subject: [PATCH 02/10] feat: postgres insert documents with embeddings --- Cargo.lock | 381 +++++++++++++++++++- Cargo.toml | 13 +- rig-postgres/Cargo.toml | 11 +- rig-postgres/src/lib.rs | 57 +++ rig-postgres/tests/integration_tests.rs | 207 ++++++++++- rig-postgres/tests/migrations/001_setup.sql | 7 +- 6 files changed, 636 insertions(+), 40 deletions(-) create mode 100644 rig-postgres/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1c25a23f28..bf66935e18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,9 +90,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arc-swap" @@ -1145,6 +1145,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitpacking" @@ -1651,6 +1654,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -2165,6 +2183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -2341,6 +2360,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -2378,6 +2403,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -2941,6 +2969,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3054,6 +3093,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -3347,6 +3397,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.1", +] + [[package]] name = "heck" version = "0.4.1" @@ -3422,6 +3481,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -4597,6 +4665,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "levenshtein" @@ -4676,9 +4747,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.164" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libm" @@ -5106,6 +5177,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -5505,6 +5593,15 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -5641,6 +5738,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -6387,6 +6495,27 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rig-postgres" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures", + "httpmock", + "log", + "rig-core", + "serde", + "serde_json", + "sqlx", + "testcontainers", + "tokio", + "tokio-stream", + "tokio-test", + "tracing", + "tracing-subscriber", + "uuid 1.11.0", +] + [[package]] name = "rig-qdrant" version = "0.1.5" @@ -6491,6 +6620,26 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rusqlite" version = "0.32.1" @@ -6500,7 +6649,7 @@ dependencies = [ "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", - "hashlink", + "hashlink 0.9.1", "libsqlite3-sys", "smallvec", ] @@ -7187,6 +7336,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "smawk" @@ -7261,6 +7413,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -7302,6 +7457,197 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "sqlx" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" +dependencies = [ + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener 5.3.1", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.1", + "hashlink 0.10.0", + "indexmap 2.6.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.3", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid 1.11.0", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.89", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.89", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.6.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.3", + "tracing", + "uuid 1.11.0", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.6.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.3", + "tracing", + "uuid 1.11.0", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "tracing", + "url", + "uuid 1.11.0", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -7852,9 +8198,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -7870,9 +8216,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -8109,6 +8455,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -8449,6 +8796,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.95" @@ -8570,6 +8923,16 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall 0.5.7", + "wasite", +] + [[package]] name = "widestring" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 6937f367c2..e1375ffff2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,13 @@ [workspace] resolver = "2" members = [ - "rig-core", "rig-lancedb", - "rig-mongodb", "rig-neo4j", - "rig-qdrant", "rig-core/rig-core-derive", - "rig-sqlite", "rig-eternalai" + "rig-core", + "rig-lancedb", + "rig-mongodb", + "rig-neo4j", + "rig-postgres", + "rig-qdrant", + "rig-core/rig-core-derive", + "rig-sqlite", + "rig-eternalai", ] diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml index e57d793bba..37020b6e43 100644 --- a/rig-postgres/Cargo.toml +++ b/rig-postgres/Cargo.toml @@ -6,11 +6,13 @@ description = "PostgreSQL-based vector store implementation for the rig framewor license = "MIT" [dependencies] -pgvector = { version = "0.4.0", features = ["postgres"] } +futures = "0.3.31" +tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] } +tokio-stream = "0.1.6" rig-core = { path = "../rig-core", version = "0.6.0", features = ["derive"] } serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.133" -tokio-postgres = "0.7.12" + tracing = "0.1.40" sqlx = { version = "0.8", features = [ "runtime-tokio", @@ -18,12 +20,13 @@ sqlx = { version = "0.8", features = [ "uuid", "json", ] } +uuid = { version = "1.11.0", features = ["v4"] } [dev-dependencies] anyhow = "1.0.94" log = "0.4.22" -tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread"] } tokio-test = "0.4.4" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -uuid = { version = "1.11.0", features = ["v4"] } + testcontainers = "0.23.1" +httpmock = "0.7.0" diff --git a/rig-postgres/src/lib.rs b/rig-postgres/src/lib.rs new file mode 100644 index 0000000000..4da316fc9d --- /dev/null +++ b/rig-postgres/src/lib.rs @@ -0,0 +1,57 @@ +use futures::{Stream, StreamExt}; +use rig::{ + embeddings::{self, Embedding, EmbeddingError, EmbeddingModel}, + vector_store::VectorStoreError, + Embed, OneOrMany, +}; +use serde::Serialize; +use sqlx::PgPool; +use uuid::Uuid; + +pub struct PostgresVectorStore { + model: Model, + pg_pool: PgPool, + documents_table: String, +} + +impl PostgresVectorStore { + pub fn new(model: Model, pg_pool: PgPool, documents_table: Option) -> Self { + Self { + model, + pg_pool, + documents_table: documents_table.unwrap_or(String::from("documents")), + } + } + + pub async fn insert_documents( + &self, + documents: Vec<(Doc, OneOrMany)>, + ) -> Result<(), VectorStoreError> { + for (document, embeddings) in documents { + let id = Uuid::new_v4(); + let json_document = serde_json::to_value(&document).unwrap(); + + for embedding in embeddings { + let embedding_text = embedding.document; + let embedding: Vec = embedding.vec; + + sqlx::query( + format!( + "INSERT INTO {} (id, document, embedded_text, embedding) VALUES ($1, $2, $3, $4)", + self.documents_table + ) + .as_str(), + ) + .bind(id) + .bind(&json_document) + .bind(&embedding_text) + .bind(&embedding) + .execute(&self.pg_pool) + .await + .map_err(|e| VectorStoreError::DatastoreError(e.into()))?; + } + } + + Ok(()) + } +} diff --git a/rig-postgres/tests/integration_tests.rs b/rig-postgres/tests/integration_tests.rs index 03b5fb7751..f06ee438f3 100644 --- a/rig-postgres/tests/integration_tests.rs +++ b/rig-postgres/tests/integration_tests.rs @@ -1,16 +1,96 @@ -use sqlx::postgres::PgPoolOptions; +use std::env; + +use rig::{embeddings::EmbeddingsBuilder, Embed}; +use rig_postgres::PostgresVectorStore; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sqlx::{postgres::PgPoolOptions, PgPool}; use testcontainers::{ core::{IntoContainerPort, WaitFor}, runners::AsyncRunner, - GenericImage, ImageExt, + ContainerAsync, GenericImage, ImageExt, }; const POSTGRES_PORT: u16 = 5432; +#[derive(Embed, Clone, Serialize, Deserialize, Debug)] +struct Word { + id: String, + #[embed] + definition: String, +} + #[tokio::test] async fn vector_search_test() { + let container = start_container().await; + + let host = container.get_host().await.unwrap().to_string(); + let port = container + .get_host_port_ipv4(POSTGRES_PORT) + .await + .expect("Error getting docker port"); + + println!("Container started on host:port {}:{}", host, port); + + // connect to Postgres + let pg_pool = connect_to_postgres(host, port).await; + + // run migrations on Postgres + sqlx::migrate!("./tests/migrations") + .run(&pg_pool) + .await + .expect("Failed to run migrations"); + + println!("Connected to postgres and ran migrations"); + + // init fake openai service + let openai_mock = create_openai_mock_service().await; + let openai_client = rig::providers::openai::Client::from_url("TEST", &openai_mock.base_url()); + + let model = openai_client.embedding_model(rig::providers::openai::TEXT_EMBEDDING_ADA_002); + + // create test documents with mocked embeddings + let words = vec![ + Word { + id: "0981d983-a5f8-49eb-89ea-f7d3b2196d2e".to_string(), + definition: "Definition of a *flurbo*: A flurbo is a green alien that lives on cold planets".to_string(), + }, + Word { + id: "62a36d43-80b6-4fd6-990c-f75bb02287d1".to_string(), + definition: "Definition of a *glarb-glarb*: A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), + }, + Word { + id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), + definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), + } + ]; + + let documents = EmbeddingsBuilder::new(model.clone()) + .documents(words) + .unwrap() + .build() + .await + .unwrap(); + + // insert documents into vector store + let vector_store = PostgresVectorStore::new(model, pg_pool.clone(), None); + + vector_store + .insert_documents(documents) + .await + .expect("Failed to insert documents"); + + let documents_count: i64 = sqlx::query_scalar("SELECT count(*) FROM documents") + .fetch_one(&pg_pool) + .await + .expect("Failed to fetch documents count"); + + assert_eq!(documents_count, 3); +} + +async fn start_container() -> ContainerAsync { // Setup a local postgres container for testing. NOTE: docker service must be running. - let container = GenericImage::new("pgvector/pgvector", "pg17") + GenericImage::new("pgvector/pgvector", "pg17") .with_wait_for(WaitFor::message_on_stderr( "database system is ready to accept connections", )) @@ -20,18 +100,12 @@ async fn vector_search_test() { .with_env_var("POSTGRES_DB", "rig") .start() .await - .expect("Failed to start postgres with pgvector container"); - - let host = container.get_host().await.unwrap().to_string(); - let port = container - .get_host_port_ipv4(POSTGRES_PORT) - .await - .expect("Error getting docker port"); - - println!("Container started on host:port {}:{}", host, port); + .expect("Failed to start postgres with pgvector container") +} +async fn connect_to_postgres(host: String, port: u16) -> PgPool { // connect to Postgres - let pg_pool = PgPoolOptions::new() + PgPoolOptions::new() .max_connections(50) .idle_timeout(std::time::Duration::from_secs(5)) .connect(&format!( @@ -39,13 +113,106 @@ async fn vector_search_test() { host, port )) .await - .expect("Failed to create postgres pool"); + .expect("Failed to create postgres pool") +} - // run migrations on Postgres - sqlx::migrate!("./tests/migrations") - .run(&pg_pool) - .await - .expect("Failed to run migrations"); +async fn create_openai_mock_service() -> httpmock::MockServer { + let server = httpmock::MockServer::start(); + + server.mock(|when, then| { + when.method(httpmock::Method::POST) + .path("/embeddings") + .header("Authorization", "Bearer TEST") + .json_body(json!({ + "input": [ + "Definition of a *flurbo*: A flurbo is a green alien that lives on cold planets", + "Definition of a *glarb-glarb*: A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.", + "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans." + ], + "model": "text-embedding-ada-002", + })); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": vec![0.0043064255; 1536], + "index": 0 + }, + { + "object": "embedding", + "embedding": vec![0.0043064255; 1536], + "index": 1 + }, + { + "object": "embedding", + "embedding": vec![0.0023064255; 1536], + "index": 2 + } + ], + "model": "text-embedding-ada-002", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + )); + }); + server.mock(|when, then| { + when.method(httpmock::Method::POST) + .path("/embeddings") + .header("Authorization", "Bearer TEST") + .json_body(json!({ + "input": [ + "What is a linglingdong?" + ], + "model": "text-embedding-ada-002", + })); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": vec![0.002; 1536], + "index": 0 + } + ], + "model": "text-embedding-ada-002", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + )); + }); + + server +} + +async fn create_test_documents(model: EmbeddingModel) -> Vec<(Work, Vec)> { + let words = vec![ + Word { + id: "0981d983-a5f8-49eb-89ea-f7d3b2196d2e".to_string(), + definition: "Definition of a *flurbo*: A flurbo is a green alien that lives on cold planets".to_string(), + }, + Word { + id: "62a36d43-80b6-4fd6-990c-f75bb02287d1".to_string(), + definition: "Definition of a *glarb-glarb*: A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), + }, + Word { + id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), + definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), + } + ]; - println!("Connected to postgres"); + EmbeddingsBuilder::new(model.clone()) + .documents(words) + .unwrap() + .build() + .await + .expect("Failed to create embeddings") } diff --git a/rig-postgres/tests/migrations/001_setup.sql b/rig-postgres/tests/migrations/001_setup.sql index 5867397e90..e9339a82ce 100644 --- a/rig-postgres/tests/migrations/001_setup.sql +++ b/rig-postgres/tests/migrations/001_setup.sql @@ -3,11 +3,12 @@ CREATE EXTENSION IF NOT EXISTS vector; -- create table with embeddings using 1536 dimensions (text-embedding-3-small) CREATE TABLE documents ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + id uuid DEFAULT gen_random_uuid(), -- we can have repeated entries document jsonb NOT NULL, - embeddings vector(1536) + embedded_text text NOT NULL, + embedding vector(1536) ); -- create index on embeddings CREATE INDEX IF NOT EXISTS document_embeddings_idx ON documents -USING hnsw(embeddings vector_cosine_ops); +USING hnsw(embedding vector_cosine_ops); From 6b7d0116ec37b9b92311f6717da42374bbabb339 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Wed, 22 Jan 2025 17:24:01 +0400 Subject: [PATCH 03/10] feat: postgres vector index --- rig-postgres/Cargo.toml | 3 +- rig-postgres/src/lib.rs | 126 +++++++++++++++++++++++- rig-postgres/tests/integration_tests.rs | 51 ++++------ 3 files changed, 143 insertions(+), 37 deletions(-) diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml index 37020b6e43..f988db7d8c 100644 --- a/rig-postgres/Cargo.toml +++ b/rig-postgres/Cargo.toml @@ -20,7 +20,8 @@ sqlx = { version = "0.8", features = [ "uuid", "json", ] } -uuid = { version = "1.11.0", features = ["v4"] } +pgvector = { version = "0.4", features = ["sqlx"] } +uuid = { version = "1.11.0", features = ["v4", "serde"] } [dev-dependencies] anyhow = "1.0.94" diff --git a/rig-postgres/src/lib.rs b/rig-postgres/src/lib.rs index 4da316fc9d..72d6c03f03 100644 --- a/rig-postgres/src/lib.rs +++ b/rig-postgres/src/lib.rs @@ -1,10 +1,12 @@ -use futures::{Stream, StreamExt}; +use std::fmt::Display; + use rig::{ - embeddings::{self, Embedding, EmbeddingError, EmbeddingModel}, - vector_store::VectorStoreError, + embeddings::{Embedding, EmbeddingModel}, + vector_store::{VectorStoreError, VectorStoreIndex}, Embed, OneOrMany, }; -use serde::Serialize; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::Value; use sqlx::PgPool; use uuid::Uuid; @@ -12,17 +14,84 @@ pub struct PostgresVectorStore { model: Model, pg_pool: PgPool, documents_table: String, + distance_function: PgVectorDistanceFunction, +} + +/* PgVector supported distances +<-> - L2 distance +<#> - (negative) inner product +<=> - cosine distance +<+> - L1 distance (added in 0.7.0) +<~> - Hamming distance (binary vectors, added in 0.7.0) +<%> - Jaccard distance (binary vectors, added in 0.7.0) + */ +pub enum PgVectorDistanceFunction { + L2, + InnerProduct, + Cosine, + L1, + Hamming, + Jaccard, +} + +impl Display for PgVectorDistanceFunction { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + PgVectorDistanceFunction::L2 => write!(f, "<->"), + PgVectorDistanceFunction::InnerProduct => write!(f, "<#>"), + PgVectorDistanceFunction::Cosine => write!(f, "<=>"), + PgVectorDistanceFunction::L1 => write!(f, "<+>"), + PgVectorDistanceFunction::Hamming => write!(f, "<~>"), + PgVectorDistanceFunction::Jaccard => write!(f, "<%>"), + } + } +} + +#[derive(Deserialize, sqlx::FromRow)] +pub struct SearchResult { + id: Uuid, + document: Value, + //embedded_text: String, + distance: f64, +} + +impl SearchResult { + pub fn into_result(self) -> Result<(f64, String, T), VectorStoreError> { + let document: T = + serde_json::from_value(self.document).map_err(VectorStoreError::JsonError)?; + Ok((self.distance, self.id.to_string(), document)) + } } impl PostgresVectorStore { - pub fn new(model: Model, pg_pool: PgPool, documents_table: Option) -> Self { + pub fn new( + model: Model, + pg_pool: PgPool, + documents_table: Option, + distance_function: PgVectorDistanceFunction, + ) -> Self { Self { model, pg_pool, documents_table: documents_table.unwrap_or(String::from("documents")), + distance_function, } } + pub fn default(model: Model, pg_pool: PgPool) -> Self { + Self::new(model, pg_pool, None, PgVectorDistanceFunction::Cosine) + } + + fn search_query(&self) -> String { + format!( + "SELECT id, document, embedded_text, embedding {} $1 as distance \ + FROM {} \ + ORDER BY distance \ + LIMIT $2", + self.distance_function, self.documents_table + ) + } + pub async fn insert_documents( &self, documents: Vec<(Doc, OneOrMany)>, @@ -55,3 +124,50 @@ impl PostgresVectorStore { Ok(()) } } + +impl VectorStoreIndex for PostgresVectorStore { + /// Get the top n documents based on the distance to the given query. + /// The result is a list of tuples of the form (score, id, document) + async fn top_n Deserialize<'a> + Send>( + &self, + query: &str, + n: usize, + ) -> Result, VectorStoreError> { + let embedded_query: pgvector::Vector = self + .model + .embed_text(query) + .await? + .vec + .iter() + .map(|&x| x as f32) + .collect::>() + .into(); + + let rows: Vec = sqlx::query_as(self.search_query().as_str()) + .bind(embedded_query) + .bind(n as i64) + .fetch_all(&self.pg_pool) + .await + .map_err(|e| VectorStoreError::DatastoreError(Box::new(e)))?; + + let rows: Vec<(f64, String, T)> = rows + .into_iter() + .flat_map(SearchResult::into_result) + .collect(); + Ok(rows) + } + + /// Same as `top_n` but returns the document ids only. + async fn top_n_ids( + &self, + query: &str, + n: usize, + ) -> Result, VectorStoreError> { + self.top_n::(query, n).await.map(|result| { + result + .into_iter() + .map(|(distance, id, _)| (distance, id)) + .collect() + }) + } +} diff --git a/rig-postgres/tests/integration_tests.rs b/rig-postgres/tests/integration_tests.rs index f06ee438f3..44c11f8077 100644 --- a/rig-postgres/tests/integration_tests.rs +++ b/rig-postgres/tests/integration_tests.rs @@ -1,6 +1,4 @@ -use std::env; - -use rig::{embeddings::EmbeddingsBuilder, Embed}; +use rig::{embeddings::EmbeddingsBuilder, vector_store::VectorStoreIndex, Embed}; use rig_postgres::PostgresVectorStore; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -13,7 +11,7 @@ use testcontainers::{ const POSTGRES_PORT: u16 = 5432; -#[derive(Embed, Clone, Serialize, Deserialize, Debug)] +#[derive(Embed, Clone, Serialize, Deserialize, Debug, PartialEq)] struct Word { id: String, #[embed] @@ -70,10 +68,10 @@ async fn vector_search_test() { .unwrap() .build() .await - .unwrap(); + .expect("Failed to create embeddings"); // insert documents into vector store - let vector_store = PostgresVectorStore::new(model, pg_pool.clone(), None); + let vector_store = PostgresVectorStore::default(model, pg_pool.clone()); vector_store .insert_documents(documents) @@ -86,6 +84,21 @@ async fn vector_search_test() { .expect("Failed to fetch documents count"); assert_eq!(documents_count, 3); + + // search for a document + let results = vector_store + .top_n::("What is a linglingdong?", 1) + .await + .expect("Failed to search for document"); + + let (score, id, doc) = results[0].clone(); + println!("Score: {}, ID: {}, Document: {:?}", score, id, doc); + + assert_eq!(results.len(), 1); + assert_eq!(doc, Word { + id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), + definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), + }); } async fn start_container() -> ContainerAsync { @@ -177,7 +190,7 @@ async fn create_openai_mock_service() -> httpmock::MockServer { "data": [ { "object": "embedding", - "embedding": vec![0.002; 1536], + "embedding": vec![0.0024; 1536], "index": 0 } ], @@ -192,27 +205,3 @@ async fn create_openai_mock_service() -> httpmock::MockServer { server } - -async fn create_test_documents(model: EmbeddingModel) -> Vec<(Work, Vec)> { - let words = vec![ - Word { - id: "0981d983-a5f8-49eb-89ea-f7d3b2196d2e".to_string(), - definition: "Definition of a *flurbo*: A flurbo is a green alien that lives on cold planets".to_string(), - }, - Word { - id: "62a36d43-80b6-4fd6-990c-f75bb02287d1".to_string(), - definition: "Definition of a *glarb-glarb*: A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), - }, - Word { - id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), - definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), - } - ]; - - EmbeddingsBuilder::new(model.clone()) - .documents(words) - .unwrap() - .build() - .await - .expect("Failed to create embeddings") -} From 8e8b9d961f0908e54e8abb9214b05b6a8f003479 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Wed, 22 Jan 2025 17:27:09 +0400 Subject: [PATCH 04/10] chore: add makefile for postgres module --- rig-postgres/Makefile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 rig-postgres/Makefile diff --git a/rig-postgres/Makefile b/rig-postgres/Makefile new file mode 100644 index 0000000000..996b730f79 --- /dev/null +++ b/rig-postgres/Makefile @@ -0,0 +1,32 @@ +# Makefile + +MAKEFLAGS += -j2 +-include .env +export + +CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +CURRENT_PATH := $(shell pwd) +DEFAULT_BRANCH := $(shell git remote show upstream | sed -n '/HEAD branch/s/.*: //p') +AMM := ${HOME}/amm + +.PHONY: gitRebase +gitRebase: + git checkout $(DEFAULT_BRANCH) && \ + git pull upstream $(DEFAULT_BRANCH) && \ + git push origin $(DEFAULT_BRANCH) && \ + git checkout $(CURRENT_BRANCH) && \ + git rebase $(DEFAULT_BRANCH) + git push --force origin $(CURRENT_BRANCH) + +.PHONY: gitAmmend +gitAmmend: + git add . && git commit --amend --no-edit && git push --force origin $(CURRENT_BRANCH) + + +.PHONY: test +test: + cargo watch -qcx 'test' + +.PHONY: fix +fix: + cargo clippy --fix From d4f46ee8dcc8b856f589bed372d5eade7fb70a81 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Wed, 22 Jan 2025 17:27:31 +0400 Subject: [PATCH 05/10] chore: update cargo lock --- Cargo.lock | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bf66935e18..33e7c4a018 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5628,6 +5628,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "pgvector" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0e8871b6d7ca78348c6cd29b911b94851f3429f0cd403130ca17f26c1fb91a6" +dependencies = [ + "sqlx", +] + [[package]] name = "phf" version = "0.11.2" @@ -6503,6 +6512,7 @@ dependencies = [ "futures", "httpmock", "log", + "pgvector", "rig-core", "serde", "serde_json", From efdffed7aff2e46baecd467d6d845407ca40fe0f Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Thu, 23 Jan 2025 11:40:12 +0400 Subject: [PATCH 06/10] feat: added example with more than one embedding per document --- Cargo.lock | 1 + rig-postgres/Cargo.toml | 2 + rig-postgres/Makefile | 4 + .../examples/migrations/001_setup.sql | 14 +++ .../examples/vector_search_postgres.rs | 99 +++++++++++++++++++ rig-postgres/src/lib.rs | 9 ++ 6 files changed, 129 insertions(+) create mode 100644 rig-postgres/examples/migrations/001_setup.sql create mode 100644 rig-postgres/examples/vector_search_postgres.rs diff --git a/Cargo.lock b/Cargo.lock index 33e7c4a018..38cbaf0e77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6509,6 +6509,7 @@ name = "rig-postgres" version = "0.1.0" dependencies = [ "anyhow", + "dotenvy", "futures", "httpmock", "log", diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml index f988db7d8c..baf9eda1a0 100644 --- a/rig-postgres/Cargo.toml +++ b/rig-postgres/Cargo.toml @@ -31,3 +31,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } testcontainers = "0.23.1" httpmock = "0.7.0" + +dotenvy = "0.15.7" diff --git a/rig-postgres/Makefile b/rig-postgres/Makefile index 996b730f79..68c296a514 100644 --- a/rig-postgres/Makefile +++ b/rig-postgres/Makefile @@ -30,3 +30,7 @@ test: .PHONY: fix fix: cargo clippy --fix + +.PHONY: run +run: + cargo run --example vector_search_postgres diff --git a/rig-postgres/examples/migrations/001_setup.sql b/rig-postgres/examples/migrations/001_setup.sql new file mode 100644 index 0000000000..e9339a82ce --- /dev/null +++ b/rig-postgres/examples/migrations/001_setup.sql @@ -0,0 +1,14 @@ +-- ensure extension is installed +CREATE EXTENSION IF NOT EXISTS vector; + +-- create table with embeddings using 1536 dimensions (text-embedding-3-small) +CREATE TABLE documents ( + id uuid DEFAULT gen_random_uuid(), -- we can have repeated entries + document jsonb NOT NULL, + embedded_text text NOT NULL, + embedding vector(1536) +); + +-- create index on embeddings +CREATE INDEX IF NOT EXISTS document_embeddings_idx ON documents +USING hnsw(embedding vector_cosine_ops); diff --git a/rig-postgres/examples/vector_search_postgres.rs b/rig-postgres/examples/vector_search_postgres.rs new file mode 100644 index 0000000000..a892ea71a9 --- /dev/null +++ b/rig-postgres/examples/vector_search_postgres.rs @@ -0,0 +1,99 @@ +use rig::{embeddings::EmbeddingsBuilder, vector_store::VectorStoreIndex, Embed}; +use rig_postgres::PostgresVectorStore; +use serde::{Deserialize, Serialize}; +use sqlx::postgres::PgPoolOptions; + +// A vector search needs to be performed on the `definitions` field, so we derive the `Embed` trait for `WordDefinition` +// and tag that field with `#[embed]`. +// We are not going to store the definitions on our database so we skip the `Serialize` trait +#[derive(Embed, Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)] +struct WordDefinition { + id: String, + word: String, + #[serde(skip)] // we don't want to serialize this field, we use only to create embeddings + #[embed] + definitions: Vec, +} + +impl std::fmt::Display for WordDefinition { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.word) + } +} + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + // load environment variables from .env file + dotenvy::dotenv().ok(); + + // Create OpenAI client + let openai_client = rig::providers::openai::Client::from_env(); + let model = openai_client.embedding_model(rig::providers::openai::TEXT_EMBEDDING_3_SMALL); + + // setup Postgres + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL not set"); + let pool = PgPoolOptions::new() + .max_connections(50) + .idle_timeout(std::time::Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("Failed to create postgres pool"); + + // make sure database is setup + sqlx::migrate!("./examples/migrations").run(&pool).await?; + + // create test documents with mocked embeddings + let words = vec![ + WordDefinition { + id: "doc0".to_string(), + word: "flurbo".to_string(), + definitions: vec![ + "1. *flurbo* (name): A flurbo is a green alien that lives on cold planets.".to_string(), + "2. *flurbo* (name): A fictional digital currency that originated in the animated series Rick and Morty.".to_string() + ] + }, + WordDefinition { + id: "doc1".to_string(), + word: "glarb-glarb".to_string(), + definitions: vec![ + "1. *glarb-glarb* (noun): A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), + "2. *glarb-glarb* (noun): A fictional creature found in the distant, swampy marshlands of the planet Glibbo in the Andromeda galaxy.".to_string() + ] + }, + WordDefinition { + id: "doc2".to_string(), + word: "linglingdong".to_string(), + definitions: vec![ + "1. *linglingdong* (noun): A term used by inhabitants of the far side of the moon to describe humans.".to_string(), + "2. *linglingdong* (noun): A rare, mystical instrument crafted by the ancient monks of the Nebulon Mountain Ranges on the planet Quarm.".to_string() + ] + }]; + + let documents = EmbeddingsBuilder::new(model.clone()) + .documents(words) + .unwrap() + .build() + .await + .expect("Failed to create embeddings"); + + // init vector store + sqlx::query("TRUNCATE documents").execute(&pool).await?; + let vector_store = PostgresVectorStore::default(model, pool); + vector_store.insert_documents(documents).await?; + + // query vector + let query = "What does \"glarb-glarb\" mean?"; + + let results = vector_store.top_n::(query, 2).await?; + + println!("#{} results for query: {}", results.len(), query); + for (distance, _id, doc) in results.iter() { + println!("Result distance {} for word: {}", distance, doc); + + // expected output as we have 2 definitions for the word "glarb-glarb" + //Result distance 0.2988138616792033 for word: glarb-glarb + //Result distance 0.3107331580360614 for word: glarb-glarb + } + + Ok(()) +} diff --git a/rig-postgres/src/lib.rs b/rig-postgres/src/lib.rs index 72d6c03f03..edc8584aaa 100644 --- a/rig-postgres/src/lib.rs +++ b/rig-postgres/src/lib.rs @@ -57,6 +57,10 @@ pub struct SearchResult { impl SearchResult { pub fn into_result(self) -> Result<(f64, String, T), VectorStoreError> { + println!( + "SearchResults into_result, id: {}, distance: {}", + self.id, self.distance + ); let document: T = serde_json::from_value(self.document).map_err(VectorStoreError::JsonError)?; Ok((self.distance, self.id.to_string(), document)) @@ -150,10 +154,15 @@ impl VectorStoreIndex for PostgresVectorStore { .await .map_err(|e| VectorStoreError::DatastoreError(Box::new(e)))?; + println!("#rows: {}", rows.len()); + let rows: Vec<(f64, String, T)> = rows .into_iter() .flat_map(SearchResult::into_result) .collect(); + + println!("#rows: {}", rows.len()); + Ok(rows) } From 19c3581c32bc46de75c4e23dc7763031d5b13fd6 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Thu, 23 Jan 2025 17:16:32 +0400 Subject: [PATCH 07/10] fix: return only nearest match when a document has more than one embedding --- rig-postgres/Cargo.toml | 2 +- .../examples/vector_search_postgres.rs | 6 ++-- rig-postgres/src/lib.rs | 18 ++++------ rig-postgres/tests/integration_tests.rs | 33 +++++++++++-------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml index baf9eda1a0..4e00ba2fb8 100644 --- a/rig-postgres/Cargo.toml +++ b/rig-postgres/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.133" tracing = "0.1.40" -sqlx = { version = "0.8", features = [ +sqlx = { version = "0.8.3", features = [ "runtime-tokio", "postgres", "uuid", diff --git a/rig-postgres/examples/vector_search_postgres.rs b/rig-postgres/examples/vector_search_postgres.rs index a892ea71a9..fc9adf1750 100644 --- a/rig-postgres/examples/vector_search_postgres.rs +++ b/rig-postgres/examples/vector_search_postgres.rs @@ -90,9 +90,9 @@ async fn main() -> Result<(), anyhow::Error> { for (distance, _id, doc) in results.iter() { println!("Result distance {} for word: {}", distance, doc); - // expected output as we have 2 definitions for the word "glarb-glarb" - //Result distance 0.2988138616792033 for word: glarb-glarb - //Result distance 0.3107331580360614 for word: glarb-glarb + // expected output (even if we have 2 entries on glarb-glarb the index only gives closest match) + // Result distance 0.2988549857990437 for word: glarb-glarb + //Result distance 0.7072261746390949 for word: linglingdong } Ok(()) diff --git a/rig-postgres/src/lib.rs b/rig-postgres/src/lib.rs index edc8584aaa..7dd2cb220d 100644 --- a/rig-postgres/src/lib.rs +++ b/rig-postgres/src/lib.rs @@ -47,7 +47,7 @@ impl Display for PgVectorDistanceFunction { } } -#[derive(Deserialize, sqlx::FromRow)] +#[derive(Debug, Deserialize, sqlx::FromRow)] pub struct SearchResult { id: Uuid, document: Value, @@ -57,10 +57,6 @@ pub struct SearchResult { impl SearchResult { pub fn into_result(self) -> Result<(f64, String, T), VectorStoreError> { - println!( - "SearchResults into_result, id: {}, distance: {}", - self.id, self.distance - ); let document: T = serde_json::from_value(self.document).map_err(VectorStoreError::JsonError)?; Ok((self.distance, self.id.to_string(), document)) @@ -88,8 +84,12 @@ impl PostgresVectorStore { fn search_query(&self) -> String { format!( - "SELECT id, document, embedded_text, embedding {} $1 as distance \ - FROM {} \ + " + SELECT id, document, distance FROM ( \ + SELECT DISTINCT ON (id) id, document, embedding {} $1 as distance \ + FROM {} \ + ORDER BY id, distance \ + ) \ ORDER BY distance \ LIMIT $2", self.distance_function, self.documents_table @@ -154,15 +154,11 @@ impl VectorStoreIndex for PostgresVectorStore { .await .map_err(|e| VectorStoreError::DatastoreError(Box::new(e)))?; - println!("#rows: {}", rows.len()); - let rows: Vec<(f64, String, T)> = rows .into_iter() .flat_map(SearchResult::into_result) .collect(); - println!("#rows: {}", rows.len()); - Ok(rows) } diff --git a/rig-postgres/tests/integration_tests.rs b/rig-postgres/tests/integration_tests.rs index 44c11f8077..c955079460 100644 --- a/rig-postgres/tests/integration_tests.rs +++ b/rig-postgres/tests/integration_tests.rs @@ -14,6 +14,7 @@ const POSTGRES_PORT: u16 = 5432; #[derive(Embed, Clone, Serialize, Deserialize, Debug, PartialEq)] struct Word { id: String, + name: String, #[embed] definition: String, } @@ -51,14 +52,17 @@ async fn vector_search_test() { let words = vec![ Word { id: "0981d983-a5f8-49eb-89ea-f7d3b2196d2e".to_string(), + name: "flurbo".to_string(), definition: "Definition of a *flurbo*: A flurbo is a green alien that lives on cold planets".to_string(), }, Word { id: "62a36d43-80b6-4fd6-990c-f75bb02287d1".to_string(), + name: "glarb-glarb".to_string(), definition: "Definition of a *glarb-glarb*: A glarb-glarb is a ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), }, Word { id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), + name: "linglingdong".to_string(), definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), } ]; @@ -87,18 +91,21 @@ async fn vector_search_test() { // search for a document let results = vector_store - .top_n::("What is a linglingdong?", 1) + .top_n::("What does \"glarb-glarb\" mean?", 1) .await .expect("Failed to search for document"); - let (score, id, doc) = results[0].clone(); - println!("Score: {}, ID: {}, Document: {:?}", score, id, doc); + assert_eq!( + results.len(), + 1, + "Expected one result, got {}", + results.len() + ); - assert_eq!(results.len(), 1); - assert_eq!(doc, Word { - id: "f9e17d59-32e5-440c-be02-b2759a654824".to_string(), - definition: "Definition of a *linglingdong*: A term used by inhabitants of the far side of the moon to describe humans.".to_string(), - }); + let (distance, id, doc) = results[0].clone(); + println!("Distance: {}, id: {}, document: {:?}", distance, id, doc); + + assert_eq!(doc.name, "glarb-glarb"); } async fn start_container() -> ContainerAsync { @@ -151,17 +158,17 @@ async fn create_openai_mock_service() -> httpmock::MockServer { "data": [ { "object": "embedding", - "embedding": vec![0.0043064255; 1536], + "embedding": vec![-0.009366372,0.027089776,-0.0069206688,0.033625767,0.018732743,-0.012323801,0.013050807,-0.0006140726,0.028190872,0.025184033,-0.016064703,0.02365944,0.0040549967,-0.025932215,-0.025240501,0.0511586,-0.024591137,-0.02483112,-0.027230943,0.03828425,0.031056544,-0.02732976,0.0026380417,0.023108892,-0.029757816,0.021570181,0.0034268075,0.012443793,0.038086616,-0.028063823,0.07843189,-0.023955889,0.003599736,-0.051215064,0.008018235,-0.026214547,-0.0054631284,0.00724888,-0.0019480924,0.017448131,-0.03591266,-0.012768474,0.027357992,0.040797006,0.008964048,-0.0032609373,0.018153962,-0.020214988,0.011582679,0.048702307,0.0016075291,0.05680524,-0.02891082,0.039554745,-0.02404059,-0.013777813,-0.013170798,0.027640324,-0.024379387,-0.02320771,-0.014681276,0.050678633,0.06250835,0.05228793,0.025889864,-0.0011046248,-0.003878539,0.039752375,0.024097055,0.035771493,-0.028021473,0.05573238,-0.012344977,-0.036053825,0.007114772,0.016107053,0.029814282,-0.02453467,-0.009811045,-0.03066128,-0.02816264,0.0087240655,0.03275054,-0.06177429,0.0047290646,0.013587238,-0.0050678635,-0.016827,-0.028444972,-0.03952651,0.016572902,-0.039639443,-0.01541534,-0.025819281,0.034218665,0.008611132,0.009387546,-0.014116611,0.012514376,0.05293729,-0.009034631,-0.03164944,0.015627088,-0.0100863185,0.03602559,-0.030350713,-0.0021598416,-0.000511286,-0.009436955,-0.0025004048,-0.027612092,0.005269025,0.042378068,0.046274252,-0.029870749,0.030266015,0.015005957,0.0342469,0.033625767,-0.05494185,0.009020515,-0.02649688,0.02720271,0.0006696567,-0.02453467,0.0028162638,0.047798846,0.027019193,-0.036533788,0.012239102,0.018605694,-0.06262129,-0.022134846,0.001674583,-0.013516655,0.004683186,-0.028840236,-0.028572021,0.0078347195,0.025184033,-0.026553346,-0.0038997137,0.03656202,0.0041502835,0.028939053,-0.014836558,-0.0057595773,0.036646724,-0.02795089,-0.0100863185,0.027188594,0.0063207126,-0.049097575,0.022445412,0.004107934,0.014208369,0.042942733,0.017066984,-0.011963828,-0.038510114,-0.033879865,-0.014751859,-0.04424146,-0.03582796,-0.010269835,-0.0027650911,-0.035263296,-0.0071924133,-0.06996193,0.042180434,-0.006144255,0.033089336,-0.051638562,-0.01895861,-0.039752375,0.046387184,0.002152783,-0.04398736,-0.027837956,0.008985223,0.020398503,0.06002383,-0.000120763194,-0.024562903,-0.0066595115,0.040091176,-0.029249618,-0.032919936,-0.027343877,0.07131712,-0.017631648,0.024068821,0.025635766,-0.015867071,-0.025946332,-0.012359093,0.027753258,-0.057059344,0.0027809723,-0.00012440264,0.02145725,0.013453131,0.00753827,-0.01412367,0.006977135,-0.011928536,0.027710907,0.047516514,-0.0023486512,-0.013911921,0.02545225,-0.0142436605,-0.046810683,0.023137126,-0.039215945,0.047036547,0.06437175,0.038961846,0.006557166,0.01795633,-0.0020892585,0.04065584,0.027936773,-0.007019485,-0.02707566,0.0044926116,0.002412176,-0.038735982,0.026482763,0.016968166,0.04825058,0.016064703,-0.009133447,-0.09429896,-0.014709509,-0.02220543,-0.005152563,-0.027965007,0.013290789,-0.011123889,-0.06256482,0.02169723,0.032468207,-0.029898982,-0.055534746,-0.009726346,-0.034472764,-0.040684074,0.052711427,0.009006398,0.044834357,-0.0021845456,0.006493641,0.02011617,-0.0017381078,-0.032609373,-0.024562903,0.033795167,0.016798768,-0.040966406,-0.0342469,-0.04048644,-0.018351594,0.020596135,-0.037465487,-0.003407397,0.024591137,-0.001570473,-0.037098452,0.020878468,0.014130727,-0.022332478,0.002415705,-0.0070865387,-0.059628565,0.014991841,0.011850895,-0.027386226,-6.7219335e-05,0.008159401,0.027400343,-0.0068465564,0.018337477,-0.05019867,-0.01441306,-0.016615251,0.046274252,0.005269025,0.027273294,-0.025762815,0.070865385,0.025042867,0.04539902,0.026892144,0.012775533,-0.0054137204,0.0065501076,0.016276453,-0.01454011,-0.00033901923,0.049746938,-0.00059466227,0.0076088533,0.04618955,-0.036420856,0.029108452,-0.02511345,0.0105804,-0.012224985,-0.0035467986,0.016601134,0.017758697,-0.016643485,0.0014575401,0.008942873,-0.049549304,0.02361709,0.055760615,0.023984123,0.03723962,-0.031790607,-0.027287409,0.03153651,-0.031931773,-0.03461393,-0.06561401,-0.026186313,0.034388065,-0.025692232,0.0151753565,0.04006294,0.008314684,-0.020130288,0.013968387,-0.03918771,-0.053219624,0.018732743,0.02444997,-0.0124085015,-0.041389905,-0.0064407038,-0.004464378,-0.024167638,0.015782371,-0.04161577,0.024435854,0.07549564,-0.027357992,0.020596135,-0.010043968,-0.009098155,0.0037973684,-0.034472764,0.022473644,-0.060249697,0.015598855,-0.005773694,-0.030887146,-0.022643045,0.036929056,0.015754137,0.028289689,0.015923537,0.0420675,-0.037606653,-0.045596655,-0.009796928,-0.007940594,0.02269951,-0.031169478,-0.029023752,0.025649883,-0.0014557756,-0.0015634147,0.005607824,0.009458129,0.03164944,-0.022487761,-0.02015852,0.010361592,0.06437175,0.07143005,-0.05536535,-0.072333515,-0.035630327,-0.044100292,-0.0066418657,0.0025780462,-0.05609941,-0.008695832,0.023504158,-0.034811564,0.05389722,-0.013671938,0.008632308,0.01962209,0.017942213,0.034218665,-0.016262336,0.052626725,0.024929935,-0.017561065,-0.035404462,-0.01949504,-0.04506022,0.0006401001,-0.0035856192,0.05276789,0.032609373,-0.04458026,-0.010799208,-0.015810605,0.02037027,0.031225944,-0.009486362,0.015443573,-0.0007375929,-0.016431736,-0.012973166,0.046980083,0.040458206,0.029334318,-0.020186754,-0.023504158,-0.028106174,0.01403897,0.019311525,0.008102935,0.041813403,-0.030633045,0.03958298,-0.03153651,-0.033061102,0.028628487,0.03277877,0.006214838,-0.02453467,0.016417619,-0.027922656,0.012930816,-0.015965886,0.0072418214,0.06871966,0.029193152,-0.004993751,-0.029842516,0.038510114,0.02453467,0.030858912,-0.05389722,-0.033061102,-0.022600694,-0.016431736,0.04048644,0.0076794364,-0.013975445,0.0016622309,-0.0019763256,0.08729713,0.012916699,0.029164918,-0.019438574,-0.035686795,-0.05773694,-0.014921258,-0.016092937,-0.016262336,0.037663117,-0.018379828,-0.012267335,0.015612972,0.008124109,0.009712229,0.05169503,0.02545225,0.025776932,0.017772814,0.0026980373,0.042547464,0.052542027,0.049379908,0.016445853,-0.032214105,0.012168519,-0.019523272,0.027753258,0.018069262,0.010608633,-0.054038387,-0.011363872,-0.029503718,0.004496141,0.056833476,-0.016728185,0.059628565,-0.044297926,-0.054010153,0.002387472,-0.0018686864,-0.026398063,-0.02795089,-0.03661849,-0.014850675,0.03565856,-0.0029133155,0.013961328,0.025903981,-0.024718186,-0.07176885,0.02711801,0.0074747456,0.010756858,0.005918389,0.034755096,-0.001295199,0.01191442,-0.010213369,0.0011875599,-0.015951771,-0.01182972,-6.6006185e-05,-0.019664438,-0.005420779,-0.014003678,0.049379908,-0.01616352,-0.022177197,-0.055760615,-0.048335277,-0.010043968,0.033230502,-0.034670398,0.018351594,-0.013869571,-0.024718186,-0.008561725,-0.039328877,-0.010403942,-0.0048667016,-0.0338234,0.0075453287,0.040260576,0.057482842,-0.008604075,0.004753769,0.05545005,-0.017377548,0.017194033,-0.0007243586,0.026158081,0.038961846,-0.03187531,-0.0041644005,-0.006306596,0.02291126,-4.8194994e-05,0.028896702,0.003649144,0.013827221,0.0062183673,-0.01778693,-0.022388944,0.043930896,0.02086435,0.043676794,-0.0014831264,-0.0009466952,0.024845235,-0.005886627,-0.035715025,-0.039300647,-0.01804103,-0.012669658,0.0082935095,-0.0133260805,-0.027936773,0.015584739,0.015358873,-0.010658042,0.0011655027,-0.0017636941,-0.030011915,-0.030717745,0.04362033,0.010792149,0.015796488,-0.03749372,-0.014427177,-0.03241174,-0.037296087,-0.0001965297,-0.034839798,-0.014328361,-0.006948902,-0.017688114,-0.010192193,0.011335638,0.0053431373,0.013057865,0.025621649,-0.031592976,-0.03537623,-0.0026856852,-0.019269174,-0.004016176,-0.03548916,0.007905303,-0.03715492,0.00061495486,-0.004058526,-0.0017231088,-0.006701861,-0.016050586,-0.024718186,0.018337477,-0.002623925,-0.009239322,-0.00766532,0.011794428,0.001368429,-0.028148523,-0.018535111,-0.00295743,0.006098376,-0.007439454,0.009740462,0.0030156611,0.008215868,-0.016445853,0.014737742,0.027061544,-0.023984123,0.0255793,0.010022794,-0.019890305,0.02624278,0.01278965,0.028148523,0.011441513,0.007961769,0.006560695,0.013396664,0.03303287,-0.016177636,0.023320641,-0.0014628338,-0.028769653,0.031677675,0.009987502,0.016968166,0.033851635,-0.0035520922,-0.036533788,0.024718186,0.006772444,-0.012930816,0.0050043385,0.0016763476,-0.009366372,-0.007785311,-0.014765975,-0.019876188,0.019678555,0.01703875,-0.006274834,0.03961121,0.021810165,0.028303806,0.014384827,-0.0023292408,0.048504677,-0.014808325,6.694362e-05,-0.02449232,-0.030463647,0.008441733,0.0061019054,-0.031931773,-0.013671938,-0.012281451,-0.06917139,0.015556506,-0.009535771,0.004824352,0.03608206,-0.0032750538,-0.04825058,0.0013419604,-0.016643485,0.004545549,-0.03136711,0.036110293,0.014582459,0.007891186,-0.026101613,-0.041672237,0.012274394,0.017109333,-0.017617531,-0.02412529,-0.0067159776,0.034755096,-0.015838837,-0.023320641,-0.027499158,-0.02361709,0.03057658,0.030294247,-0.0119355945,0.009782812,0.022459527,0.041107573,0.0013251968,0.06115316,-0.011794428,0.060419098,0.013290789,-0.010883907,0.0057136985,0.008582899,0.02329241,-0.0027421517,0.016643485,0.03574326,0.02020087,-0.04712125,0.033258736,1.9630912e-05,-0.0508198,-0.021429015,0.015754137,-0.019847956,-0.016615251,0.00065774587,-0.0056042946,0.018097496,-0.033625767,-0.019099776,-0.012443793,-0.01454011,0.017603414,0.014236602,0.0070053684,-0.017151682,-0.014130727,-0.019509157,-0.0044996697,-0.017518714,-0.036110293,0.015711788,-0.007418279,0.013149623,0.010029852,0.011385047,-0.02611573,0.007418279,-0.002316889,-0.019156242,0.006864202,0.025819281,-0.012902583,0.027315643,-0.006906552,0.0049055223,-0.018760975,0.011321522,-0.009189914,-0.038086616,-0.008949932,-0.019269174,-0.015048307,-0.004760827,-0.0068994937,-0.029503718,0.0017381078,-0.008349976,-0.0008950814,0.03723962,0.008907582,0.03715492,-0.008942873,0.018295128,-0.014893025,-0.028275572,0.0020698481,-0.011300347,0.016827,0.0017222266,-0.057793405,-0.014977724,-0.037973683,-0.009542829,-0.022713628,-0.027174477,0.010552167,0.030689513,0.010227485,-0.065670475,-0.005078451,-0.017758697,-0.00039438283,0.014963608,0.023490041,-0.006557166,0.019480923,0.013671938,0.028713187,0.007884127,-0.029814282,0.036844354,-0.009493421,0.034218665,-0.009027573,0.035771493,0.0021086687,0.014525993,-0.029560184,-0.049012873,-0.024464088,-0.0041820463,0.02970135,-0.039470043,0.019424457,-0.03935711,0.002128079,0.025692232,-0.00061495486,-0.018168079,-0.0010366887,-0.0037091395,0.027005076,-0.00027659736,-0.0046796566,-0.007940594,0.0105733415,0.010043968,0.03537623,0.023560625,-0.047685914,-0.029588416,0.022360712,0.02169723,-0.028529672,0.034642164,0.0077429614,0.031931773,-0.005011397,-0.01016396,-0.0424063,0.017984563,-0.026722744,-0.041135807,-0.0072912294,-0.012712008,0.01712345,-0.027103893,-0.029475484,0.02253011,0.013876629,0.011363872,0.021061983,-0.033315204,-0.0334846,-0.0065748114,0.003486803,-0.0013322552,-0.055675913,-0.03520683,0.030774212,0.025918098,0.069736056,-0.024760537,-0.007552387,-0.0011152123,-0.035460927,-0.0019886775,-0.054179553,0.013375489,0.033315204,0.002574517,-0.038256016,0.018930376,0.027569741,-0.046500117,0.0068183234,-0.014441294,-0.0101286685,0.041051105,-0.0072912294,0.03574326,-0.007714728,-0.026186313,0.021894865,-0.05745461,0.04949284,0.027386226,-0.01962209,0.02320771,0.008251159,-0.011243881,0.017194033,-0.014991841,-0.0012422617,-0.03128241,0.031000078,0.009726346,-0.014709509,-0.005847806,0.030040149,0.015852954,0.060927294,0.016925817,-0.013855454,0.018026913,0.008427617,0.0055760616,-0.006959489,0.028939053,0.032214105,-0.025085218,0.02237483,0.0011716788,-0.003391516,0.01141328,0.0028974344,-0.01395427,-0.01582472,-0.019833839,0.013749579,-0.027965007,0.0041467547,0.024548788,-0.019367991,-0.039837077,-0.0044008535,0.004316154,0.034049265,0.027597975,0.018718626,-0.011730904,0.0026768623,-0.005410191,0.0091969725,0.024351154,-0.009528712,-0.034924496,-0.00285332,0.0151753565,0.008653482,-0.042886265,0.054631285,0.011773254,0.0077500194,0.0069630183,0.0045667235,-0.0509045,0.014391885,0.026906261,0.029193152,0.018520994,-0.077076696,-0.019438574,0.028741421,-0.012175577,0.05799104,-0.03286347,0.0075029787,-0.02899552,0.011582679,0.036900822,0.013862512,0.0050255135,0.0060136765,0.0059113307,0.036336157,0.013820162,0.0069665476,0.03515036,-0.006694803,0.015852954,0.0031885896,0.005607824,0.044467326,-0.0037514893,-0.022021914,-0.0023998239,0.020440852,-0.012761417,0.013883687,-0.023108892,-0.024421737,-0.00011844719,0.017843397,0.017024633,0.040091176,0.0022974785,-0.009069922,-0.004047938,-0.0046090735,0.0005549593,0.0105451085,-0.012267335,0.007068893,0.003315639,-0.0028303806,-0.02495817,-0.027965007,-0.041502837,-0.0033315201,0.031141244,-0.026369829,0.020426737,0.030378947,0.018535111,-0.024082938,-0.049605772,0.024548788,0.019509157,-0.0087240655,-0.0066277487,-0.006835969,0.030266015,0.037437253,0.006352475,-0.017066984,0.0004482024,-0.006236013,-0.030633045,-0.022798328,0.01695405,0.008978165,-0.024929935,-0.0012819647,0.02190898,0.011265055,0.0013119625,-0.007160651,-0.009458129,0.0017654587,0.03469863,-0.014864791,-0.007488862,-0.017589297,0.026750978,-0.008625249,-0.0007265643,-0.02444997,-0.006416,-0.029588416,-0.031903543,0.025932215,0.017292848,-0.01149798,-0.000648923,0.032468207,0.006680686,0.008893465,0.0077994275,-7.449821e-05,0.0033103453,-0.028063823,-0.022501878,0.0018007503,0.0033050515,-0.0022921846,-0.006193663,-0.0003132123,0.013862512,-0.0115050385,0.042886265,0.0006083377,-0.00017502392,-0.005230204,0.013467247,0.008604075,0.0075029787,0.006327771,0.00441497,-0.019255057,0.023687674,0.03698552,-0.03015308,0.010481584,0.014709509,-0.009098155,0.001772517,0.033428136,-0.019438574,0.0005068746,0.03515036,0.028374389,0.013304906,0.007735903,-0.02070907,-0.038764216,-0.0058089853,-0.01778693,0.02628513,-0.0070794807,8.453424e-05,-0.008399384,-0.003924418,-0.01712345,-0.02365944,-0.019156242,0.0061583715,-0.0100863185,-0.014935374,0.009754579,-0.0042843916,0.015485923,-0.011194472,0.017434014,-0.009895745,0.0024333508,0.034585696,-0.013820162,-0.0061795465,0.013050807,0.034557465,0.0022498348,0.013269614,-0.0039385343,-0.001080803,0.029334318,0.00803941,-0.0039103013,0.012846116,-0.023179475,0.01782928,-0.020285571,-0.016389385,-0.008568783,-0.004485553,0.06521875,0.01919859,0.008208809,-0.040091176,0.025875749,0.0039350055,0.056297045,0.0071041845,-0.025424017,-0.004891406,-0.044890825,0.008201751,-0.004129109,0.09412956,-0.008575841,-0.021146683,-0.011377988,0.029108452,-0.01412367,0.0065395203,0.016798768,-0.0057242857,0.016770534,-0.013290789,0.0043020374,0.019240942,-0.0015378284,0.017052867,0.028148523,-0.0069418433,0.01233086,-0.014314244,-0.008074702,0.010778032,0.018182196,-0.046641283,0.0020892585,-0.015457689,0.017405782,0.06166136,-0.012464968,-8.139109e-05,-0.03627969,-0.009535771,0.004097346,0.03057658,0.021132566,0.034642164,-0.018153962,-0.007100655,-0.017137567,0.029983683,-0.013311964,-0.04731888,0.008229984,-0.011060365,-0.025042867,0.013290789,0.010474525,0.020087937,-0.012352034,-0.032468207,-0.0056007653,0.008032352,-0.014794208,-0.0008761122,0.01703875,0.026920378,-0.03435983,-0.0119920615,0.007982943,0.049605772,0.0074323956,-0.019565623,0.0151471235,-0.028600255,-0.0004918757,0.011018015,0.0052902,0.013333139,0.0026980373,0.041841634,0.0011760902,0.0062783626,-0.015316523,-0.0032838767,-0.00066436303,0.0071924133,0.0077711944,-0.0065006996,-0.015133007,-0.007368871,0.03749372,-0.0012387326,0.018944493,0.005681936,-0.012168519,-0.010643925,-0.029136686,-0.022276012,-0.0058548646,-0.028939053,0.019226825,0.041164037,0.002932726,0.03023778,-0.00092022656,0.032665838,0.0064477623,-0.054574817,-0.011166239,0.0012652013,0.0073759295,0.008554666,-0.024520554,-0.01919859,-0.0055090077,-0.005307846,0.001558121,0.00080155884,0.01703875,0.029475484,0.0013754873,0.0014319537,-0.008519375,0.00878759,-0.009902802,0.052316163,0.028106174,0.052174997,0.029673116,0.0030209548,0.0032768184,-0.0015925302,-0.022445412,0.015090657,-0.03757842,-0.0064265872,-0.018139845,0.0061019054,-0.013184914,0.034303363,-0.036929056,0.014406002,0.0008792002,0.052259695,-0.022981843,0.0051772674,0.01095449,0.009253439,-0.021753699,-0.0062924796,0.0058795684,-0.001928682,-0.015895303,0.043055665,-0.03723962,0.04898464,-0.027682675,0.019480923,-0.017391665,-0.024464088,0.02070907,-0.0033932803,0.016897583,0.0048102355,0.024689954,0.002948607,0.009253439,-0.017801046,0.028063823,-0.009938094,0.022318361,-0.008032352,0.0073618125,0.035291526,-0.0066383365,0.019523272,0.019339757,-0.014751859,-0.012069702,-0.0020945522,0.025169918,0.01616352,-0.0019939714,-0.000612308,-0.022727745,-0.0050502177,-0.00044224696,0.01187207,0.044664957,-0.042293366,0.0057913396,-0.020483203,-0.029390784,0.01953739,0.03952651,-0.012500259,0.009069922,-0.009528712,-0.010029852,-0.0064089415,0.031225944,0.015500039,-0.017546948,0.011406221,0.02790854,-0.0084135,-0.036477324,-0.04723418,-0.029249618,0.015951771,0.007090068,-0.01053805,-0.0012493201,0.008088818,0.0040620547,-0.046980083,0.029193152,-0.00078567764,0.025056984,-0.035263296,-0.013523714,-0.028713187,0.0070971265,-0.019593855,-0.01812573,0.024421737,-0.027343877,-0.011780312,0.020779653,-0.0119355945,0.028473204,-0.011215648,-0.009500479,0.038848914,0.00891464,0.003906772,0.008702891,0.031847075,0.0045314324,0.016248219,-0.009253439,0.007990002,0.027357992,-0.021189034,0.0008637601,0.022755977,0.0084982,0.010834499,0.004827881,-0.031423576,0.013255497,-0.012521434,-0.0034268075,-0.030124849,0.011554446,-0.0027650911,0.015768254,0.03515036,-0.036138523,-0.0013172562,0.02137255,-0.018972725,-6.1208746e-05,-0.0042491,-0.030124849,0.07329345,-0.030435413,-0.00029247854,0.003345637,-0.01979149,-0.0054666577,0.013495481,-0.025551066,0.017165799,0.025720466,-0.01957974,0.013820162,-0.04856114,0.023955889,-0.04853291,0.027710907,0.0062254253,0.016587019,0.008568783,0.006384237,-0.0031091836,-0.019466806,-0.039441813,-0.033315204,-0.023955889,0.026214547,0.021061983,-0.0005249615,0.0023221825,0.011060365,-0.009803986,-0.027880307,0.011123889,0.023320641,-0.021217266,0.011681496,0.029023752,-0.0029115512,0.027428575,-0.0075170957,-0.02741446,0.010686275,-0.00978987,0.032665838,-0.011617971,-0.001178737,0.00063215947,-0.037973683,-0.010947432,-0.0128955245,0.0010472761,0.010135727,-0.0128390575,-0.017095216,0.005149034,-0.0038044266,-0.0045420197,0.006853615,-0.01582472,-0.029729582,0.007898244,-0.0009431661,-0.004753769,0.0005254026,0.012267335,0.028233223,0.047036547,0.006003089,0.011568563,-0.0075876783,-0.0421522,0.028670838,0.026143964,0.0062713046,0.02045497,0.04661305,0.004411441,0.026906261,-0.04808118,-0.02495817,0.02111845,-0.029051986,-0.0003039483,0.024689954,0.019170359,-0.034303363,-0.020511435,-0.023602974,0.004993751,0.0036667897,-0.029334318,0.04641542,0.019297408,-0.013551947,-7.493935e-05,-0.00891464,0.045850754,0.00050996256,-0.011130948,0.007418279,-0.012387326,0.0077994275,0.0005170209,-0.0029344906,0.020737302,-0.0038961847,-0.03241174,0.02795089,-0.013544888,0.028529672,0.0070053684,0.046104852,0.018069262,-0.008822882,-0.0065006996,0.010192193,0.004478495,-0.026440412,0.012980224,-0.012888466,0.022643045,-0.035432693,-0.0021968975,0.0038750097,-0.030181315,-0.007637087,0.0128390575,-0.023885306,0.017109333,0.0060595553,0.02186663,0.024972284,-0.059346233,0.027612092,0.02799324], "index": 0 }, { "object": "embedding", - "embedding": vec![0.0043064255; 1536], + "embedding": vec![-0.016489647,0.061171718,-0.020970378,0.036124155,-0.010408655,0.025381532,0.027594067,-0.039352506,-0.002556939,0.025284125,-0.03208871,-0.033842042,0.036959074,-0.0055209007,-0.037849654,0.0059975004,-0.012725554,-0.022556724,0.0012019352,0.016517477,-0.0018542155,0.009733762,-0.047813017,0.0682407,-0.061450027,0.027886288,0.03626331,0.040270917,0.036708597,0.011187912,0.03946383,-0.027023539,0.005875741,-0.028971683,0.0147919785,0.038851555,0.017714195,-0.0074377353,-0.011069632,0.004981682,-0.050902218,-0.014061425,-0.009079742,0.018994404,0.057052787,-0.0070237545,0.0073264125,-0.010840029,-0.022668047,0.002868294,-0.06846334,0.021165192,-0.050484758,0.08304659,-0.018883081,-0.013998806,0.0050756107,0.018799588,-0.012537697,-0.05198761,-0.06356515,0.04736773,0.038740233,0.0016472252,-0.016336579,-0.0050025554,-0.04394456,0.03732087,-0.009907704,0.026091214,0.0011714955,0.022445401,-0.045586567,-0.016267002,-0.029945755,0.018980488,-0.009698974,-0.012843834,0.030725013,-0.008001305,-0.030502368,0.02550677,-0.010499104,-0.018771758,-0.01618351,-0.0098450845,0.008717944,-0.011501007,-0.046477146,0.020705987,0.017101921,-0.024407461,0.00069533085,0.007556015,-0.015404252,-0.01712975,-0.019495355,-0.0050582164,0.0047625164,0.017602872,0.0018159484,0.014666741,-0.007799533,-0.036680765,0.05805469,-0.014527587,-0.019411862,-0.0098450845,0.023864763,-0.013108226,-0.0729719,0.021179108,0.012628147,0.041745942,-0.012043704,0.023419473,0.04394456,0.006251455,0.033062786,0.0028735122,-0.020010222,-0.016823614,0.021165192,-0.055577762,0.034760453,0.01977366,-0.0033014081,0.009685058,-0.04255303,0.008829267,0.033034954,-0.01838213,0.001748981,-0.013525685,0.018159484,-0.035734527,-0.049566347,-0.049148887,-0.011306193,0.03932468,-0.06985488,0.037793994,0.022417571,0.0072081327,0.009065827,0.022320163,-0.0044285487,0.016406154,-0.043332286,-0.008898843,0.0011836714,0.027093116,-0.027886288,-0.0110348435,-0.030418877,-0.017004514,0.006835898,0.01086786,0.0044702943,-0.0011949776,0.0048564444,-0.01244029,0.022542808,0.002095994,-0.011173997,0.04469599,-0.033396754,0.013546558,-0.075254016,0.02257064,-0.02446312,0.06740578,-0.020455511,-0.015724305,-0.05989151,-0.015640812,-0.006794152,-0.051709305,-0.020539002,-0.03292363,0.016253086,0.027204439,0.020344188,-0.04892624,-0.03423167,0.031142471,-0.029027345,-0.031198133,0.004842529,0.04166245,-0.013790076,0.04519694,0.059501883,-0.019912815,0.017435888,-0.014144916,0.0487036,-0.05905659,0.024184816,-0.00077229994,0.022765454,-0.013205633,0.0061923144,-0.025812907,0.017199328,-0.016698377,0.01948144,-0.01333087,0.020928632,-0.013546558,0.039157692,-0.03506659,-0.028721208,0.003691037,-0.06634822,0.048592277,0.021791382,0.04875926,0.0017672449,0.0011688863,0.00074751326,0.060670767,0.03459347,0.009587651,-0.033786383,-0.01422145,0.014117086,-0.0342595,0.04892624,0.05332348,0.07002186,0.026967878,-0.0048912326,-0.061950978,-0.009344134,0.024838835,0.0353449,0.016934937,0.012753385,0.004400718,-0.010457358,0.015153777,0.025687668,0.0013202154,-0.032255698,0.013741372,0.004846008,-0.013790076,0.018521283,0.0019098768,-0.011368811,-0.02684264,0.030335383,0.01930054,-0.03014057,-0.015849542,-0.021206938,-0.013365659,0.036625106,-0.037042566,-0.0181734,-0.010923522,0.007764745,0.010478231,-0.024546614,0.028582055,0.02924999,-0.010916564,-0.014805893,-0.00240561,0.002250802,0.010262544,0.013198676,-0.06796239,-0.0586113,-0.0046337997,-0.022069687,-0.00079708657,-0.008244824,0.034342993,0.022445401,0.040493563,0.01820123,0.014527587,-0.019634508,0.02446312,0.035623204,-0.041022345,0.021346092,-0.03506659,0.050289944,-0.0050025554,0.026926132,0.032478344,-0.037988808,-0.03556754,-0.008745775,0.052878194,0.017408058,0.01966234,0.052432902,0.010714792,-0.03904637,0.01733848,-0.046477146,0.049900316,-0.025117142,-0.0057922495,-0.008356146,-0.0284429,0.016684461,-0.016503561,-0.00048051318,-0.0144023495,-0.014082298,-0.034537807,0.0006461924,0.059279237,-0.001303691,0.0055174218,0.029556127,0.0063419044,0.055577762,-0.02506148,-0.015070285,-0.05276687,-0.028108934,-0.03055803,-0.01600261,0.006227103,0.066849165,0.036541615,0.0031709522,0.03233919,0.048647936,-0.043916732,0.026286028,0.0057087573,0.03233919,-0.063954785,-0.030725013,-0.017241074,0.005733109,0.0006079253,-0.047673866,0.073417194,0.037988808,0.014158832,0.033368923,0.0129134115,0.017867263,-0.008557918,-0.056023054,-0.0030422355,-0.069910534,-0.010930479,0.001805512,-0.014945047,0.06389912,0.09529207,-0.011716695,0.007917814,-0.006247976,0.049093228,-0.041968588,-0.027677558,0.036569446,-0.012655978,0.07981824,-0.008933632,-0.04004827,0.029194329,-0.030780675,-0.010220798,0.021638313,0.013476982,-0.016921021,0.011960212,-0.011445345,-0.02131826,0.025339786,0.010227756,-0.020886887,-0.06707181,0.0022542807,-0.04828614,-0.018493451,0.014444096,-0.0030718055,0.012391587,0.008237866,-0.020441595,0.012273307,-0.020956462,-0.0024247435,0.008425723,0.024407461,-0.037960976,0.010506062,0.07030016,-0.014889386,-0.011723652,-0.053713113,-0.03011274,-0.014179705,0.038378436,0.0052356366,0.057498075,0.022765454,-0.04344361,0.027204439,0.002233408,0.033897705,0.0064462693,-0.0039345548,-0.024282223,-0.0010853945,0.00038941135,-0.013247379,0.042747844,0.06679351,-0.0033901182,-0.0074099046,-0.02660608,0.007667338,0.047172915,-0.002200359,0.047757357,0.047312066,-0.0042511285,-0.013720499,0.023238573,-0.005555689,-0.04041007,0.009281514,-0.0066549988,0.004505083,0.0095459055,-0.026480842,-0.002158613,0.0008262218,0.009803339,0.014471927,0.04210774,-0.0005561777,0.029973585,0.033619396,0.047117252,0.011800186,-0.05182063,-0.06562462,-0.03765484,-0.026466927,0.038378436,0.014151874,0.009135404,0.026759148,0.013671796,0.03105898,-0.03592934,0.040270917,-0.013991848,-0.021304345,-0.020650325,0.011793229,-0.028609885,-0.018688267,0.06384346,-0.002556939,0.003833669,-0.016545309,-0.033619396,0.0017689842,0.010255586,0.007521227,0.0510692,0.015139861,-0.007924771,0.023168998,0.0018072514,0.0493437,0.03317411,0.019857153,-0.007722999,-0.03768267,0.013762245,0.029416973,-0.012405503,-0.014750233,-0.03367506,-0.036569446,-0.0015602546,0.050790895,-0.016239172,0.03940817,0.007382074,-0.06723879,-0.038378436,0.015348591,-0.0057887705,-0.0127673,-0.02657825,-0.01885525,0.0011297496,0.0013906617,0.010074687,-0.0018594338,-0.022765454,0.0021812255,-0.0078691095,-0.035289235,0.016517477,0.0021777465,-0.006808067,0.014569334,-0.0048529655,-0.031198133,0.029583957,-0.0038754148,-0.01161233,-0.0022960268,-0.025965976,-0.03322977,-0.03556754,0.036012832,0.023990002,-0.025284125,-0.0135117695,-0.032478344,-0.016433986,0.023781272,-0.014332773,0.0013610916,0.007855195,0.0018159484,-0.012962115,0.0041015386,-0.009511117,-0.010060772,-0.0044563795,-0.024212645,-0.010840029,0.04113367,-0.023906508,0.010881775,0.009316303,-0.0028822094,-0.0107356645,0.0141936205,0.03896288,0.051264018,0.010645215,0.02508931,-0.0028021964,-0.003607545,-0.008982335,-0.0037258253,0.029194329,-0.033396754,0.01977366,-0.030446706,-0.026049467,-0.008738817,-0.040187426,0.026202535,0.008077839,-0.016934937,0.024254393,0.019258793,-0.008453554,-0.0072498783,0.029416973,-0.0022107954,-0.0012784695,-0.028971683,-0.039853457,0.022904607,0.04046573,-0.03194956,-0.024365714,-0.004153721,-0.0706898,0.006595859,-0.0067558847,-0.03590151,0.025200633,-0.011716695,-0.005673969,-0.017630702,-0.021220854,0.019495355,-0.0110278865,-0.045029957,0.01767245,-0.001847258,-0.003263141,0.02084514,-0.0073681585,0.015306845,0.021206938,-0.021540906,-0.0041119754,-0.028053273,0.010366909,-0.031782575,-0.03988129,-0.024657937,-0.037460025,0.0033274994,-0.012628147,-0.014778064,0.02972311,-0.039157692,-0.0332576,-0.04884275,-0.0001932924,-0.01416579,-0.01712975,0.0065471553,-0.022403656,0.01948144,0.023628203,-0.015390337,-0.0125168245,-0.015195522,0.036207646,-0.04427853,-0.0048599234,-0.0315321,-0.025325872,-0.0013558734,-0.026272113,0.01861869,-0.016934937,-0.0030857208,-0.03865674,-0.025492854,0.060559444,0.0015628637,0.0077508297,0.010575638,0.0069646146,0.027315762,-0.054854166,0.012328968,-0.030391045,-0.006933305,0.01443018,-0.02131826,-0.0038267113,0.035734527,0.0080639245,0.013386532,-0.02060858,0.016698377,0.0023812582,-0.030307554,0.0025917273,-0.03515008,0.010819157,-0.020719903,-0.011000056,0.009469371,-0.013977933,-0.007834322,0.04038224,0.028136764,0.027983695,-0.011097463,-0.025604177,0.06857467,-0.035289235,0.009852042,0.005600914,-0.007827364,-0.011932381,-0.019230964,-0.023461219,0.0011375769,-0.014666741,0.0038093172,0.011201828,-0.0017211504,-0.01072175,0.036875583,0.012982988,0.027566237,0.013323913,-0.0051591024,0.0061679627,-0.01250291,0.0005305213,0.040549226,-0.028220257,-0.03631897,-0.019871067,0.015779966,-0.00291004,-0.009587651,-0.026272113,-0.008182204,0.010109476,-0.010881775,-0.029222159,0.029583957,-0.038267113,0.042803504,0.008029136,-0.00055487314,-0.034621302,0.028971683,0.011674948,-0.044250697,0.04892624,-0.011118336,0.00614709,0.0043137474,-0.045002125,-9.343916e-05,0.045558736,-0.020622496,0.01600261,-0.025812907,0.010109476,-0.009615482,-0.017533295,0.0061923144,-0.01903615,-0.04522477,-0.040938854,0.02815068,0.0054513244,-0.038795896,0.028053273,-0.0110348435,0.036151987,-0.01754721,-0.02461619,-0.020135459,-6.1586135e-05,0.049510688,0.04784085,-0.03812796,-0.010234714,-0.019829322,0.052154597,0.012308095,-0.029973585,0.00697853,0.049844652,-0.04004827,0.012558571,0.004501604,-0.025938146,-0.026327774,-0.024059577,-0.01950927,-0.04121716,0.027607983,0.03848976,-0.0006170572,0.019648423,0.014346688,0.048675768,-0.029750941,0.0018194272,0.0038058383,-0.015098115,0.014833724,0.0018855251,-0.018743929,0.00409806,-0.010262544,-0.0048912326,0.028943853,0.007562973,-0.020622496,0.014360604,0.039658643,-0.021749636,-0.030307554,0.0069924453,-0.01134098,0.008933632,0.0019377074,-0.029138668,-0.0107356645,-0.0109374365,-0.05399142,0.021540906,-0.030001417,0.010617385,-0.0342595,0.007771703,0.041578956,0.023210743,0.00021057784,-0.06746144,0.012822961,-0.0008062185,0.043081813,-0.004035441,0.062396266,0.0066863084,0.020455511,0.015807796,0.011695821,-0.00028613364,-0.037126057,0.0013776161,0.019829322,0.027190523,-0.033842042,-0.0042093825,0.018215146,0.018465621,-0.00142545,-0.012155026,-0.0037362617,0.010923522,-0.015459914,0.010805241,-0.008140459,-0.012127196,0.022946352,-0.0009766811,0.0061992723,-0.0035327503,0.0032492257,-0.020803394,0.036124155,-0.021415668,-0.03912986,-0.0032927112,0.00920498,-0.0042859167,0.021360006,-0.017296735,-0.04923238,-0.0016307009,0.017602872,0.020775564,-0.008404849,-0.015139861,0.0083631035,-0.0029674408,-0.0049190633,-0.0055904775,0.024838835,0.041328482,-0.023113336,-0.052878194,-0.015459914,0.016086103,-0.022083603,-0.03250617,0.002830027,-0.010192967,0.0018211667,-0.035233576,-0.018604774,-0.04035441,-0.03548405,-0.003510138,-0.0092536835,0.019370116,0.006940263,-0.040604886,-0.020872971,0.0019829322,0.046755455,0.0025082354,0.04041007,-0.034871776,0.021693975,0.039714307,-0.006470621,0.00083187484,0.02110953,-0.003694516,-0.024365714,0.017004514,-0.00088797096,-0.019370116,-0.022737622,-0.009608524,0.02039985,0.040966682,-0.013657881,0.014235366,0.014583249,0.00084709475,0.009427625,-0.020483341,0.017825518,-0.005534816,0.018423876,0.031810407,0.0019446651,-0.0051417085,0.015960865,0.040131766,-0.053462636,0.002628255,0.05282253,0.0044076755,0.012760343,0.022848945,0.045419585,-0.009991195,-0.00095928693,-0.015223353,-0.006251455,0.023029845,0.011320108,-0.006794152,-0.005597435,-0.014569334,-0.01339349,-0.028999513,0.011730609,-0.005882699,-0.0067245755,0.0034144702,0.0025882483,0.0019011798,0.020483341,-0.0013158668,-0.015306845,-0.01642007,0.011173997,0.026439097,-0.021568736,-0.019077895,0.031754747,-0.052238088,0.037070397,-0.014458011,0.04302615,0.011459261,-0.013553516,0.006046204,-0.008474426,0.01728282,0.0031883463,-0.03812796,0.0073751165,-0.0030248414,-0.037599176,-0.016447902,0.03158776,0.037404362,0.024156984,-0.00091580156,0.0104364855,-0.023990002,0.0044111544,-0.012308095,-0.0042128614,-0.006227103,-0.024630105,-0.026425181,0.02877687,-0.012732512,-0.013442193,-0.02749666,0.0051799756,-0.0098450845,0.02616079,0.04703376,0.011452303,-0.024518782,0.02238974,0.051124863,-0.0017168019,0.02354471,0.007931729,0.024699682,-0.016767953,-0.03501093,0.03450998,-0.0008605752,0.029890094,0.019147472,-0.05822167,-0.0033727242,0.056524005,-0.0033988154,0.0070689796,-0.03414818,-0.029973585,-0.017352397,0.00433462,0.016614884,0.04514128,0.032673158,-0.0016020006,0.018952657,-0.021610482,-0.02393434,0.003224874,-0.012523782,-0.0076395073,0.029361311,0.012948199,-0.02613296,-0.03940817,0.0016385282,-0.01579388,-0.010979183,-0.03932468,-0.0052182428,0.068852976,-0.008084797,-0.022306249,-0.028178511,-0.036179814,0.043499272,-0.0021899226,-0.026091214,0.00045268255,0.03506659,0.028609885,-0.038044468,-0.049009737,0.0025360659,-0.013560473,-0.0035118773,0.0010323423,0.021860957,0.0024282222,-0.009984238,0.029500466,0.0058896565,-0.017268904,0.009594609,0.008648368,-0.009872915,-0.024532698,0.0021360007,-0.03284014,0.0019133557,0.008077839,0.011605372,0.001375007,0.006220145,-0.013977933,-0.023628203,-0.031559933,0.0029639618,0.017408058,0.019857153,0.014124043,-0.0043520145,0.027844543,-0.01778377,-0.024045663,-0.02155482,-0.0058513894,-0.013108226,0.00066402135,0.0068915593,-7.604501e-05,-0.0011001795,0.011981085,-0.014875471,-0.022793284,0.0055800406,0.009705931,0.035706695,-0.013428278,0.0069924453,-0.039630815,-0.00034896997,-0.0042059035,-0.0021655706,0.0062410184,-0.015946949,-0.004198946,0.006366256,-0.011208786,-0.03242268,0.02746883,-0.0041258903,0.009483286,-0.01072175,-0.02110953,-0.009177149,0.016614884,0.005844432,-0.000923629,0.008933632,-0.0035971086,-0.016587054,-0.0048738387,0.010680003,-0.004463337,0.047200743,0.020441595,0.008237866,-0.023614287,0.038044468,-0.0044494215,0.018688267,-0.0073055397,-0.03183824,-0.0031257274,-0.021276515,0.01571039,-0.012788174,-0.01859086,-0.028053273,0.03183824,-0.01757504,0.020859055,-0.0054130573,0.009698974,-0.02238974,-0.019161386,0.024894496,-0.03400903,0.022765454,0.015334676,-0.02930565,0.042887,-0.002339512,0.0022090562,0.02529804,-0.021206938,0.022751538,-0.03690341,-0.036151987,-0.0005205197,-0.046310164,0.04469599,0.0145971645,-0.013344786,-0.009483286,0.012113281,0.025381532,0.010053814,0.014861555,-0.014972878,0.024811005,-0.04976116,0.037460025,0.002182965,0.09401186,0.005740067,-0.028526394,-0.009796381,0.004087623,-0.00593836,0.011375769,-0.013407405,0.011097463,0.0022977663,-0.00409806,0.015014623,-0.0046164053,-0.04029875,0.0049955975,-0.0089545045,-0.023823017,0.016364409,0.015251184,-0.023642119,0.0018942221,-0.012328968,-0.0073124976,0.018270807,-0.025604177,-0.035734527,0.04528043,0.013623092,0.002280372,0.020956462,0.0002678698,0.004842529,-0.028804699,0.01571039,-0.0209008,-0.021819212,0.02330815,-0.040215258,-0.011974128,-0.005360875,-0.017004514,-0.01223156,0.0006831549,0.021276515,0.0060670767,0.00033483724,0.029917924,-0.005903572,-0.033897705,-0.0065019303,0.0043276628,-0.009490244,-0.015432083,0.0022351472,0.046337996,-0.049510688,-0.00810567,-0.041523296,0.009935534,0.006021852,-0.026411265,0.029416973,-0.014903301,-0.029138668,0.001161059,-0.02309942,-0.026884386,0.019342287,0.0033361965,0.013337828,-0.016781868,0.018702181,-0.0043102684,3.9340657e-06,0.009705931,-0.008537045,0.0013784858,-0.010227756,0.004011089,0.049816824,-0.01443018,0.011306193,0.028289832,-0.006122738,-0.025993805,-0.04344361,-0.010206883,-0.013449151,-0.04169028,-0.004397239,0.016726207,-0.045920536,0.012822961,0.025687668,0.019203134,0.011445345,-0.006815025,0.01175844,0.011647117,-0.002796978,-0.0009940752,-0.008627495,-0.050763063,-0.00018481276,0.0020907759,0.0101442635,-0.0035292716,0.013490897,-0.0044876887,0.0043728873,0.0043067895,0.021930534,-0.025381532,-0.026244283,0.02215318,0.013845737,-0.018256892,0.022946352,0.021457413,-0.027580151,0.022793284,0.0014367561,-0.0069854874,-0.008773605,0.0009123228,-0.045447417,-0.0013419581,-0.009497202,0.024936242,-0.022125348,-0.004842529,-0.024922326,0.00929543,-0.022723708,0.00012545525,-0.0014854597,0.013073437,-0.038768064,-0.014082298,-0.0042476496,0.004560744,0.00587922,0.020274613,-0.018994404,0.018159484,-0.018910911,0.015960865,9.208024e-05,-0.021373922,-0.015042454,-0.005259989,-0.0068672076,-0.037070397,0.009149319,0.014945047,-0.00543393,0.014068382,0.027802797,-0.025325872,-0.022278417,-0.009838127,0.0144023495,-0.0012784695,-0.008098712,0.03503876,0.046783283,-0.0019933686,-0.005816601,-0.01754721,-0.007354243,0.018479537,-0.044334188,-0.0064114807,0.001713323,-0.034760453,-0.03186607,0.024421375,0.031754747,-0.045725722,0.00784128,-0.018660435,-0.028317664,0.018799588,-0.011354896,-0.01072175,0.0018994403,0.05198761,0.00204903,-0.016169595,-0.0020699028,0.041940756,-0.0094554555,-0.015445998,0.01840996,0.0010871339,-0.0038162747,-0.006808067,-0.01728282,0.03142078,-0.014541503,-0.013456108,0.0012010655,0.0043067895,0.026856557,-0.02152699,0.03606849,-0.014972878,0.02456053,-0.01552949,-0.03684775,0.004056314,-0.030057078,0.012308095,0.003317063,-0.0179925,0.002304724,-0.05532729,0.013365659,-0.009991195,0.057275433,0.020775564,-0.023238573,0.026244283,0.0072081327,-0.02479709,-0.025103226,0.04745122,-0.0002054683,0.016698377,0.00878752,-0.01840996,0.01728282,0.02123477,-0.004219819,0.008258739,0.0030683267,-0.014931132,0.0107356645,-0.05401925,0.015515575,0.031253796,0.040771868,0.015557321,0.021471329,-0.008530088,0.021332176,-0.01297603,-0.023168998,0.007848237,0.028971683,-0.019787576,-0.0018194272,0.033368923,-0.04121716,0.034342993,-0.018465621,-0.008001305,-0.0029326524,0.003607545,-3.8511716e-05,-0.0014750232,0.017936839,0.03862891,0.024129154,-0.019105725,0.030697182,-0.037042566,-0.010833072,-0.0031726914,-0.02328032,-0.017825518,0.009128446,-0.004992119,0.007389032,-0.018256892,-0.011841932,-0.055995222,0.029138668,-0.0054060994,0.013637007,0.037960976,0.0034753496,0.05140317,0.010227756,0.010352993,-0.026244283,-0.015515575,0.022278417,0.005555689,0.023836933,0.015863458,-0.00026156442,0.013734415,-0.0005009513,0.022208842,-0.0030300596,-0.009065827,0.004992119,-0.0042476496,0.02301593,-0.010415613,0.016712291,0.043694086,-0.0032005222,-0.008356146,0.021151276,-0.009142362,0.03899071,-0.03317411,0.0034805678,0.02833158,-0.008321358,-0.00843268,-0.0047764312,0.021721805,-0.0005466109,0.0152929295,0.016461816,0.0016611406,0.04032658,0.016948853,0.00614709,0.013560473,0.0036458122,-0.013038649,0.016503561,0.01505637,-0.02616079,0.019077895,0.0034944832,-0.02687047,0.011250531,-0.008898843,0.013476982,-0.008676198,-0.0061783995,0.0007557755,0.033786383,-0.0024247435,-0.006693266,0.0045850957,-0.04221906,-0.0055800406,0.019342287,0.009782465,-0.011793229,0.016406154,0.008446596,0.025965976,-0.023878679,-0.009803339,-0.015209438,-0.0026908738,0.0014680657,0.008307442,-0.00054791546,-0.011862805,-0.016684461,-0.0018890038,-0.03715389,-0.018771758,0.00697853,-0.013776161,0.00021188239,0.04472382,0.041996416,0.0056496174,0.0028578576,0.016086103,0.02238974,0.0048947115,-0.01754721,-0.00665152,-0.020525089,0.015028539,-0.034315165,-0.028484648,-0.002628255,0.013490897,-0.007702126,-0.015821712,-0.0057018,-0.033146277,-0.0332576,-0.0013297822,0.03556754,-0.023002014,0.017853348,0.014346688], "index": 1 }, { "object": "embedding", - "embedding": vec![0.0023064255; 1536], + "embedding": vec![0.018444104,-0.0065429183,-0.01915854,0.008463875,0.02739642,0.025267694,-0.027571384,0.02777551,-0.07686745,-0.021360168,-0.011547613,0.035342693,-0.023124387,0.014813605,0.0035211472,0.024276232,-0.07167685,-0.042982783,0.027090235,0.031318527,0.008048337,0.007348481,0.013771112,0.045811366,-0.012539075,0.029729271,0.013311832,0.061645593,0.027061073,-0.03382634,0.07815051,-0.01867739,-0.0015783196,-0.0076327976,-0.022672398,0.03700485,0.046890307,0.010410348,-0.046890307,-0.03557598,-0.029029416,-0.037529744,0.008478456,0.021374749,0.011795479,-0.00959385,-0.0827579,0.019318923,-0.006528338,0.015017729,0.015353077,-0.011926701,-0.025792586,0.050593708,0.025821745,-0.02092276,-0.011234136,0.059079457,0.02639038,-0.040037557,-0.073309846,-0.011707997,0.0034774062,0.049660567,0.004709443,-0.00638618,-0.05843792,0.022439113,0.0024513162,0.051206082,-0.0069912635,0.027090235,-0.029743852,0.027629705,0.013268091,-0.004971889,-0.034176268,0.0007659224,-0.018546166,0.011226846,0.0020211968,0.014339744,-0.011248717,-0.01781715,0.0058868043,-0.010315577,-0.043565996,0.017277678,-0.003557598,-0.032601595,-0.0075088646,-0.039191898,-0.0010516056,-0.033884663,-0.017394321,0.017481804,0.007895243,0.044032566,0.015076051,0.044061728,0.015557202,-0.008398264,-0.0071443566,-0.040358324,0.036188353,0.024596998,-0.010082291,-0.0053436873,-0.024903186,0.030851956,-0.040941536,0.048290018,-0.021097723,0.042982783,0.030851956,0.027585965,0.0627537,0.00030527543,0.011737158,0.013669049,-0.019727172,0.004301194,0.041262306,-0.008813803,0.0022745298,0.014252262,-0.03245579,0.006666851,-0.044849064,0.018021274,0.038929455,0.0070896805,-0.0026171673,-0.057300657,-0.034934446,-0.020908179,-0.039920915,5.001306e-06,-0.041699715,-0.015382238,-0.012714039,-0.047735967,0.021024821,-0.017890053,0.024247073,-0.009644882,-0.022191247,-0.006787139,0.0013386556,-0.0035466629,-0.023780502,-0.034176268,-0.029889656,0.020106262,0.012437013,-0.030297905,0.0023948175,0.012159986,0.01136536,0.028227499,-0.03589675,-0.0105269905,-0.0006670496,-0.021228945,0.017015234,-0.017321419,-0.03700485,0.0068563954,-0.077917226,-0.014930247,0.006287763,0.022526594,-0.022001702,0.006418986,-0.0032623466,-0.033330612,-0.0075744763,-0.055055287,-0.03965847,-0.015775906,0.054209627,0.012721329,0.017000653,0.007392222,-0.01777341,-0.0061784103,-0.021083143,0.009017928,0.025326015,0.03709233,-0.038929455,0.045024026,0.049427282,0.018633649,-0.005970641,-0.030035459,-0.005792032,-0.018458685,0.047356877,-0.0630453,-0.0026171673,-0.0013377443,0.02325561,-0.058991972,0.0044433526,-0.0056790346,-0.014886507,0.033097327,0.033330612,-0.01546972,0.02189964,0.02685695,-0.01412833,0.0501563,-0.04875659,0.028548267,0.067885965,0.021826738,-0.0209665,-0.016169574,0.05065203,0.03965847,0.009885456,0.039046098,0.041116502,0.011839219,-0.014660511,-0.015980031,0.05053539,0.018779451,0.020820696,-0.0018116047,-0.008259751,-0.035342693,0.012582816,0.028635748,-0.023066066,-0.006739753,-0.043886762,-0.019289764,0.013464925,-0.030676993,0.02233705,0.037034012,-0.005759226,-0.0050265654,0.0034755836,0.02599671,0.059604347,-0.031056082,-0.019348085,-0.0030053684,0.030881118,-0.0028267596,-0.020864438,-0.040474966,-0.057708904,-0.02647786,0.013297251,0.0001732552,-0.015848808,-0.007239129,-0.05065203,0.05336397,-0.009980229,0.0068162996,0.0067470428,-0.0044907383,-0.00960114,0.024028366,-0.042691175,-0.039920915,0.014427226,-0.059837632,-0.034380395,-0.025792586,-0.008908575,-0.011489292,0.033155646,0.018414944,0.017875472,0.020295804,-0.0044105467,-0.022205828,-0.06205384,-0.018473266,0.05840876,-0.03152265,0.008893995,-0.016956912,0.048464984,-0.03965847,0.005471265,0.0036979336,-0.087481916,-0.05657164,-0.003287862,-0.03335977,-0.016213316,0.012560945,-0.0073995125,0.01369821,0.013049386,-0.009469918,-0.037529744,0.047590163,-0.03776303,-0.037588064,-0.002471364,-0.017190197,0.054676197,0.0069584576,-0.010745696,-0.02045619,0.010067711,-0.017000653,-0.051235244,0.062228806,0.023007745,-0.031493492,0.068819106,0.007348481,0.029437665,0.0031529942,0.008449296,-0.016986072,0.045315634,-0.029248122,-0.053159844,-0.02277446,0.028766971,-0.024042947,0.013034806,0.06893575,-0.031726778,-0.030035459,-0.020674894,0.029000256,-0.024349134,0.0086971605,-0.029452246,0.043624315,0.011037302,-0.020383287,-0.045344796,0.061295666,0.019800074,0.033913825,0.0019701656,0.034672,-0.020762375,-0.026215415,0.02143307,-0.020427028,0.042049643,-0.03790883,0.034001306,-0.023518056,-0.0075744763,0.033476412,0.027585965,-0.022089185,0.022934843,0.01778799,-0.06491158,-0.020150002,0.018837772,-0.014398065,0.004968244,-0.022234987,-0.027308939,0.06893575,-0.031085242,0.034059625,0.022686977,0.0740097,0.042720336,0.006429921,0.004702153,-0.0017705975,0.043974243,0.0006820856,0.011686127,-0.046744507,0.002099566,0.018589908,0.011000851,0.0048479564,0.0074942843,-0.018429523,0.022964004,-0.047823448,0.059516866,0.035809264,-0.018400364,0.021695515,0.02554472,-0.014091879,-0.005241625,0.049952175,0.042137124,-0.010862338,-0.053918023,-0.004016878,0.0058321278,0.032543276,0.01139452,0.035430178,0.0013860416,-0.010155193,0.014390776,0.0063132783,0.03330145,0.034088787,0.022570334,-0.031231046,-0.00046543113,0.0024185106,0.027133975,-0.0066850767,-0.0051577883,-0.019508468,-0.015542621,-0.027921312,0.04686115,0.05697989,-0.037034012,-0.00570455,0.07739234,-0.032484952,-0.014390776,-0.042166285,0.0044907383,0.010228095,-0.012437013,-0.009623011,-0.04508235,0.035925906,-0.0062403767,0.014368905,-0.0024476713,0.028519105,-0.024101269,0.020689474,-0.0164466,0.015309336,-0.009222052,0.0109644,0.044761583,-0.060770772,-0.053568095,-0.0137055,0.022657817,-0.017204776,0.057767227,-0.022220407,0.01279423,-0.00071079057,-0.0129254535,-0.00073220546,0.022118345,-0.013880464,-0.030268744,-0.050010495,-0.037588064,0.018633649,-0.030706154,0.023940885,0.013406604,-0.019727172,-0.024276232,-0.040795736,-0.022424532,0.019289764,0.008070207,0.03230999,0.024509517,0.009542819,0.028956516,-0.006976683,0.029277282,-0.016854849,0.025909228,-0.025107311,-0.00035835692,0.0012247468,0.0077858907,0.039833434,-0.029423086,-0.016257057,-0.048610788,-0.0023346737,0.100429244,-0.018152498,0.021287266,0.044469975,-0.0069220066,-0.01369821,-0.01686943,-0.001238416,-0.0017377918,-0.0027520354,0.039337702,-0.006597595,0.030122941,0.026273737,0.0356343,0.035780106,-0.007202678,-0.001572852,0.029977137,0.007873373,-0.014959408,-0.035196893,0.022278728,-0.019858396,-0.0038054634,0.027469322,-0.013552407,-0.023415994,0.034730323,-0.014157491,-0.010548862,-0.021331009,0.03522605,-0.009892747,-0.032426633,0.03965847,-0.023211868,-0.03878365,0.0016986072,-0.052372508,-0.0041772616,-0.0034045046,-0.0136544695,-0.038987774,-0.017671347,-0.026142513,0.0037945283,0.0026317476,0.012240179,0.042662017,0.0042282925,-0.053743057,7.204728e-05,0.042107962,0.019304343,-0.0015127081,0.010104162,0.013435764,0.039483506,0.0117881885,-0.009200182,0.008930446,0.043478515,0.0029962556,0.002797599,0.036771566,0.0069548124,-0.0019701656,0.010782147,-0.0057081953,0.005788387,0.0013623487,0.005883159,-0.035401016,-0.030647833,0.021957962,-0.033184808,0.0027666157,-0.025413496,-0.024188751,0.012342241,0.029146058,-0.027673448,0.010978981,0.049077354,0.029029416,0.00089805655,-0.028314982,0.0088575445,-0.032776557,0.014988569,-0.01279423,-0.036246676,-0.012211018,-0.0010461379,-0.004457933,-0.0028869032,-0.006397115,-0.008318072,-0.023999207,-0.0164466,0.027104814,-0.030181263,-0.030502029,0.020368706,-0.020849857,0.062228806,-0.0010716534,-0.010752986,-0.00097323634,-0.0042064223,0.015863389,-0.0067288177,-0.04499487,0.026492441,-0.0113434885,0.0031949126,0.012881712,-0.033039004,0.0073193205,-0.008923155,-0.019041898,0.0034719387,0.007982725,-0.0053473325,0.0084128445,-0.021782998,0.0060070916,0.015688425,-0.008879415,0.0018790388,-0.011489292,0.013486796,0.032951523,-0.050302103,-0.017496383,-0.03700485,-0.00016562332,-0.008303492,-0.03522605,6.219418e-05,-0.039191898,-0.04006672,0.025719684,-0.00064244535,0.013982527,0.036188353,0.0007937162,0.020616572,-0.0028340498,0.01549888,-0.026259156,-0.0055842623,-0.02051451,-0.004888052,-0.0328932,0.007071455,0.00020230193,0.05654248,0.015396818,0.053101525,-0.0083763935,0.015440559,-0.010118742,-0.06677786,0.02280362,-0.040037557,0.0007486083,-0.028402463,0.0015938112,0.022001702,0.01737974,0.017700508,0.04654038,0.022016283,0.031493492,0.0036159193,0.030735314,0.116467595,-0.0070896805,0.004924503,0.05231419,0.021141464,-0.026200835,-0.031289365,-0.017627606,0.011110203,-0.022614077,0.0032076703,-0.006619465,0.015338496,-0.0005371937,0.04508235,0.017685927,0.04339103,0.026302896,-0.013734661,-0.003054577,0.0050921766,-0.023474315,0.025442658,-0.045665562,0.0127650695,-0.026769467,0.028387882,-0.008536777,-0.023313932,-0.0036614828,-0.013129578,-0.04199132,-0.026171673,-0.0040460387,-0.019945877,-0.011081043,0.0072209034,-0.032776557,0.021068562,0.0052707857,0.030647833,0.0013350105,-0.036304995,-0.004621961,0.007151647,-0.0019154896,-0.008719031,-0.029495986,-0.022949424,-0.0004911745,-0.013873174,0.0105269905,-0.0039877174,-0.014193941,0.00034833295,-0.019216862,0.0025533785,0.02601129,0.01646118,0.020645732,0.017336,0.014704253,-0.0064153406,-0.003878365,-0.01369821,0.0024075753,-0.005463975,0.02415959,-0.008369103,0.0080045955,0.033942983,0.0017031636,-0.012553656,-0.016986072,0.0149156675,0.037558902,0.016111253,-0.057708904,-0.03700485,0.02832956,-0.010133322,0.014317874,-0.027250618,-0.017977534,0.001053428,-0.032601595,0.003034529,-0.000551774,0.012991065,-0.023605539,0.0054822,-0.02461158,0.013501376,0.076517515,0.013187899,0.009207472,-0.033418093,0.017627606,0.022686977,-0.014223102,-0.011321618,-0.029248122,-0.0025406207,0.009338695,0.007778601,-0.0031876224,-0.009681332,0.03411795,0.023911724,0.016140414,-0.008303492,-0.022905683,-0.0071844524,-0.042428732,0.010388478,-0.04607381,-0.03338893,0.007858792,-0.009725073,0.033797182,-0.01826914,0.026827788,-0.014959408,-0.0095136585,0.029014837,-0.012247468,-0.0069985534,-0.07727569,-0.026259156,-0.0137784025,0.020354126,-0.016854849,0.020412447,-0.03913358,0.0063643097,0.0034573583,-0.009535529,0.025865488,-0.010184353,-0.008901285,0.031289365,0.03245579,-0.0022089183,-0.015615523,0.06287034,-0.013071257,-0.016606985,0.029525148,-0.028956516,-0.021505972,-0.005562392,-0.01729226,0.00909083,0.012947324,0.035459336,0.040737413,-0.0032350083,0.025165632,-0.03432207,-0.027133975,0.04117482,0.00102609,-0.019216862,-0.010293705,-0.024596998,0.026507022,-0.007924404,0.04887323,-0.042662017,-0.007705699,0.011737158,-0.0048078606,0.02420333,-0.027804669,0.022220407,0.02327019,-0.009827135,0.00058913603,0.0110956235,0.033418093,-0.045024026,-0.024509517,0.011912121,0.019698013,0.022657817,-0.0102426745,-0.009900037,0.009214762,-0.007938984,-0.056717444,-0.0021214364,-0.014478258,-0.013946076,0.0018553457,-0.022701558,0.016519502,-0.030502029,0.009316824,-0.00957198,0.009440757,0.035955068,-0.004796925,0.010388478,-0.0040861345,-0.012531784,-0.0005886804,-0.034351233,-0.026113352,0.03607171,-0.02138933,0.011022721,0.015076051,0.01367634,-0.05732982,0.006568434,-0.020222904,-0.018035855,0.013479506,-0.0065064677,0.009637591,-0.005726421,0.0052707857,0.008813803,-0.07634255,-0.00864613,-0.03009378,0.020004198,0.025457239,-0.013486796,-0.033651378,0.006561144,0.002079518,-0.015309336,-0.044761583,0.012415142,0.02509273,-0.022876522,0.050943635,0.027469322,-0.012189147,0.04922316,-0.031289365,0.018342042,0.040270843,0.069344,-0.022278728,0.0072245486,-0.023022326,-0.036188353,-0.053743057,0.030618671,-0.042282928,-0.004600091,0.017656768,0.034467876,-0.02732352,-0.008296202,-0.013246221,0.004519899,-0.024028366,-0.0056863246,0.033097327,0.0014434516,-0.013669049,0.014944828,-0.06362852,0.030181263,0.0016266169,0.0015537153,0.03516773,-0.015688425,-0.022512013,0.0049390835,0.012393272,0.01275778,-0.003061867,-0.03228083,0.022745298,0.004472513,0.0046292515,0.029568888,-0.006440856,-0.0062257964,0.029043997,0.0014352503,-0.020820696,0.016213316,-0.0024276231,0.0069657476,0.016650725,-0.005737356,-0.008748191,0.009426177,-0.016315378,-0.0493398,-0.029787593,-0.0013860416,-0.02873781,0.010876918,0.004844311,-0.0050375005,-0.043740958,-0.041145664,0.032776557,0.014470967,0.01416478,0.059079457,-0.012714039,0.02286194,-0.036304995,0.02375134,0.03099776,0.058583725,0.007705699,-0.023343092,-0.016971491,0.029029416,0.005981576,0.010709245,-0.0464529,-0.032689076,0.029466826,-0.021083143,0.018750291,0.022220407,0.02879613,-0.008828384,0.01692775,-0.012947324,0.0027629705,0.03429291,-0.026550762,-0.030006299,0.0025807165,-0.020835277,-0.010067711,-0.0051140473,-0.013953366,0.0011527565,-0.0088210935,-0.017627606,-0.011030012,-0.00034742168,-0.019698013,0.004337645,0.01775883,-0.027133975,-0.0014434516,-0.014777154,0.008245171,0.029029416,0.00821601,0.01143097,0.010140613,-0.013931495,-0.0008515818,0.016111253,-0.012196437,-0.04155391,0.024349134,0.014033558,-0.018094176,0.0020321321,-0.0004829731,-0.006105509,0.020164583,0.024684481,-0.02458242,-0.024742803,0.0058940942,0.0064554363,-0.016825689,0.0067251725,-0.0132097695,0.025938388,-0.008872124,-0.0003321579,0.010111452,-0.02188506,0.005515006,-0.008682581,0.032776557,0.028621167,0.013049386,0.009032508,0.0091054095,-0.00034947204,0.00915644,0.025646782,0.0010743873,-0.011438261,-0.009484498,0.0007568097,-0.0102426745,0.01643202,0.011926701,-0.02558846,-0.0065793693,-0.060362525,0.0328932,0.004290259,-0.0025697814,-0.022234987,0.024028366,-0.009338695,-0.024757383,0.029466826,-0.0073776417,-0.01094253,0.009440757,0.03513857,-0.029248122,0.0046183164,-0.0023966401,-0.011030012,0.022118345,0.00728287,-0.017583866,0.019377245,0.0039767823,-0.016781949,-0.009805265,-0.004250163,-0.0140627185,0.019391825,0.00064107846,-0.013268091,0.005923255,0.024261652,0.007432318,0.0070459396,0.039833434,-0.0028996612,-0.017510964,0.022614077,-0.02369302,-0.022205828,-0.019887555,0.028548267,0.016606985,0.008113948,-0.029116899,0.03700485,-0.019552208,-0.00052534713,0.010410348,-0.0059086746,-0.010519701,-0.034555357,-0.02233705,-0.019041898,0.0077421498,-0.006047188,-0.013829433,-0.006105509,0.037325617,0.010694665,0.053101525,-0.025253113,0.029758433,-0.025296854,-0.023488896,-0.0026098771,-0.009528239,0.014718832,0.002737455,-0.019391825,-0.023430575,0.001150934,0.032776557,-0.0013751064,-0.025807166,0.0063205687,0.047823448,-0.02286194,0.051614333,0.0014862814,0.093197405,-0.009236633,0.00091491506,0.012159986,0.006335149,-0.003122011,-0.03653828,-0.004866182,0.021083143,-0.007239129,0.018006695,0.021680936,-0.030152101,0.029466826,-0.005383783,-8.5089836e-05,-0.013253511,-0.0009026129,-0.017190197,0.022016283,-0.004425127,0.013180609,-0.03184342,0.011627805,-0.021170624,-0.028037954,0.001961053,0.03099776,0.007559896,0.024436615,0.008915866,0.027002752,-0.0007581766,0.0048479564,0.005296301,-0.009623011,0.0076765385,-0.046744507,-0.007115196,-0.015775906,0.01549888,-0.012196437,-0.03006462,0.022949424,0.010804017,0.008478456,0.026419539,-0.007661958,0.0010999029,0.015542621,0.02512189,-0.017423483,0.020674894,0.013340993,0.009972938,-0.025603041,-0.022016283,0.014376195,-0.010133322,0.02866491,-0.019129379,0.04155391,0.0069001364,0.01962511,-9.904365e-05,-0.032689076,-0.0032678142,0.0040241685,0.015192693,0.061004058,-0.0008256106,0.013603439,-0.001481725,-0.019027317,0.012903583,0.023955466,0.013071257,-0.035838425,-0.008084787,0.00728287,0.011766318,0.022643236,0.016490342,0.0075890566,-0.010760276,-0.007516155,-0.047094434,0.004975534,-0.035925906,0.0043631606,-0.01781715,-0.035750944,0.014193941,0.017029813,0.009338695,-0.012568235,-0.014973989,0.018079596,0.00049527525,-0.005318172,0.013836724,0.012123536,0.010271835,0.0039330414,0.02694443,0.011168525,0.010694665,0.018779451,-0.02369302,0.009484498,-0.02783383,0.014296004,0.01818166,-0.04184552,0.03557598,-0.0031621067,-0.02461158,0.027338099,0.0053655575,-0.0012283919,0.008850254,-0.0065866592,0.028985675,0.022234987,-0.02411585,-0.031056082,-0.013027515,0.008842964,0.015819646,-0.007960855,0.032135025,-0.015717585,0.014441807,0.0005713663,-0.009382436,0.008937736,0.0137055,-0.050477065,0.028402463,0.032484952,0.0050046947,0.0097979745,0.0060326075,-0.006652271,0.03137685,-0.0064918874,0.0237076,-0.01006042,-0.0009723251,0.023299351,0.0009404306,0.012808811,0.007938984,-0.018619068,-0.012211018,0.011576774,-0.008908575,-0.009907328,-0.029000256,-0.011620515,0.021170624,-0.0068709757,-0.009929198,0.0026080548,0.013690921,0.03093944,-0.03102692,-0.0043194196,0.0009632124,-0.0007021335,0.00913457,0.017962953,-0.015732165,-0.009484498,0.022249568,-0.0051140473,-0.014288713,-0.0057701617,-0.017831732,-0.052168384,-0.03516773,-0.022001702,0.016694466,0.01590713,-0.003358941,-0.025734264,-0.026186254,-0.020704053,0.015630104,-0.047094434,0.034030467,-8.303948e-05,-0.0005645318,0.012867132,-0.006521048,-0.009732364,0.027236037,0.004108005,0.011226846,0.013771112,-0.008500326,-0.014434516,-0.018531587,0.02777551,0.0026171673,0.04417837,-0.008719031,0.040358324,-0.015703004,-0.013610728,-0.030035459,-0.017744249,0.015148953,0.024946926,-0.0082233,-0.0027793734,-0.034263752,0.001097169,-0.0049135676,0.052139223,0.015848808,-0.005562392,0.0010789436,0.005551457,-0.031056082,0.0056644543,0.006010737,-0.0014944827,0.011307038,0.0111320745,-0.023926305,0.004873472,-0.0029816753,-0.0061747655,0.0059925113,0.016359119,0.011744448,0.0020776954,-0.037208974,0.0105269905,0.050506227,0.012947324,0.03828792,0.0056899698,0.023022326,0.05928358,0.051818457,0.016679885,-0.00456364,0.03744226,-0.00867529,0.029656371,0.025646782,-0.0093897255,0.035809264,-0.024305394,-0.0058175474,-0.016665306,0.0023401412,-0.00036769744,-0.031260207,0.012043344,-0.0013313654,0.0037945283,-0.03682989,0.024188751,-0.03604255,0.008850254,0.01592171,0.014944828,0.014762574,0.022599496,0.026959011,0.018065017,-0.0059633506,0.013428475,-0.010315577,-0.0041225855,-0.02095192,0.017729668,0.014747993,0.010067711,0.0024531388,0.0010762098,-0.018531587,-0.0182254,-0.015221854,0.03475948,0.024305394,0.0074213827,-0.00773486,-0.0016658015,0.005066661,-0.0017833554,0.028052535,0.0059414804,0.00067206164,0.0034573583,-0.012225598,0.007669248,-0.026550762,-0.0058321278,-0.034409553,0.034642838,0.028562846,-0.0016184155,-0.019435566,0.027090235,0.0022234987,0.014398065,0.006743398,-0.024815705,0.009768814,-0.00085249305,0.012808811,0.002258127,0.005197884,0.011970443,-0.04925232,0.024290813,0.0056717442,-0.0018826838,0.018283721,-0.0068272348,0.0025242178,0.024057528,0.0019410051,-0.01781715,-0.005471265,-0.019887555,-0.011839219,0.001437984,-0.024451196,-0.026507022,-0.019348085,0.010439509,0.009309534,0.008121238,0.0019482953,-0.013005645,0.049106516,-0.03280572,0.016067512,0.012648427,-0.0009440757,-0.026594503,-0.030706154,0.034788642,0.009972938,-0.022249568,0.054967806,-0.002575249,-0.022614077,-0.012451593,0.0113434885,-0.000551774,-0.016184155,-0.017015234,-0.026098773,-0.001434339,-0.044411656,-0.005318172,0.026302896,0.020820696,0.039483506,0.020339547,-0.023328511,-0.00084975926,0.006619465,-0.021287266,-0.038025472,-0.02647786,0.0013413894,-0.00228911,0.015382238,-0.017525544,0.010913369,-0.0047932803,-0.017642187,-0.011358069,0.0022563045,0.023634698,-0.0032550565,-0.013537827,0.019493887,0.020893598,-0.04059161,0.008274332,-0.004589156], "index": 2 } ], @@ -179,7 +186,7 @@ async fn create_openai_mock_service() -> httpmock::MockServer { .header("Authorization", "Bearer TEST") .json_body(json!({ "input": [ - "What is a linglingdong?" + "What does \"glarb-glarb\" mean?" ], "model": "text-embedding-ada-002", })); @@ -190,7 +197,7 @@ async fn create_openai_mock_service() -> httpmock::MockServer { "data": [ { "object": "embedding", - "embedding": vec![0.0024; 1536], + "embedding": vec![-0.018881548, 0.03401518, -0.034554653, 0.024971912, -0.013926916, 0.027655078, 0.037138443, -0.053634953, -0.024233686, 0.018327879, -0.02430467, -0.036712542, 0.058660567, -0.027797045, -0.05085241, -0.0029830716, -0.0030309854, 0.005117893, -0.021082029, 0.006157798, -0.003320242, -0.017405096, -0.060420953, 0.07586691, -0.05900129, 0.003210218, 0.04784272, 0.0041880123, 0.043072645, 0.0045571253, -0.0220474, -0.03960866, 0.004720387, -0.047360033, 0.005199524, 0.03205604, -0.0049226894, -0.0041099307, -0.0023176034, 0.028918581, -0.04168137, -0.031232635, -0.0037976045, 0.007282883, 0.022970183, -0.018626008, 0.006118757, -0.0042554466, -0.03446947, 0.006143601, -0.06592926, 0.021820255, -0.03608789, 0.066099614, -0.04699092, -0.020968456, 0.0046706987, 0.01666687, -0.02243071, -0.0148922885, -0.03583235, 0.012060056, 0.021351766, -0.024446636, -0.009731805, -0.03299302, -0.05266958, 0.050994374, 0.013124805, 0.075980484, 0.020258622, 0.0008779742, -0.030125296, -0.028364912, -0.031402994, 0.04000617, 0.016354544, 0.0057531935, 0.04270353, -0.02705882, 0.014608355, 0.0028482033, -0.013160297, 0.036542185, -0.0035935277, -0.028222945, -0.0061010113, -0.003133911, -0.001055876, 0.014565766, 0.026774887, 0.0141398655, 0.0044967895, 0.00054878934, 0.014665142, -0.026746493, -0.012606627, 0.022672053, -0.0049688285, 0.0084257135, 0.02769767, 0.031119062, 0.032481942, -0.045656435, 0.0466502, -0.026774887, -0.034554653, -0.0075810123, 0.039835807, -0.04295907, -0.08290845, -0.00441161, 0.016425528, 0.06257884, 0.036542185, 0.0534362, 0.023055363, 0.029500645, 0.04491821, 0.008447008, -0.022842415, -0.021621501, 0.014750322, -0.077116214, -0.03659897, 0.018043945, 0.00788624, 0.0053663347, -0.008319238, -0.04278871, 0.038188994, -0.039722234, -0.020074066, -0.0068179425, 0.018540828, -0.03336213, -0.027285965, -0.031033883, -0.06644034, 0.035264485, -0.032538727, 0.047047704, 0.025624957, -0.0028890187, 0.0036449905, 0.015204615, 0.0034923765, 0.01700759, -0.029245105, 0.009298807, -0.014565766, 0.03444108, -0.0326523, -0.0040850868, -0.018725384, -0.04784272, 0.021962222, -0.00016592338, -0.03040923, -0.0064985175, -0.0020425434, -1.4917577e-5, 0.04801308, 0.0031569805, 0.009660822, 0.012684709, -0.01323128, 0.0046884445, -0.042135663, -0.0007346768, -0.0831356, 0.052300468, 0.0068463357, -0.013692671, -0.028677238, -0.028435895, 0.0013353726, -0.04931917, -0.009114251, -0.053890493, -0.0028996663, 0.009021972, -0.009469167, -0.050227758, -0.041794945, 0.017433489, -0.029415464, -0.023396084, -0.035463236, 0.05372013, -0.0048765503, 0.043413363, 0.031147456, -0.010370655, 0.013600393, -0.042050485, 0.04026171, -0.018696992, 0.01790198, -0.029074743, 0.004067341, -0.026036661, 0.014118571, -0.03935312, -0.017760012, 0.0014542696, 0.024091719, -0.014622552, -0.0003693348, 0.02633479, 0.022089992, -0.032169614, 0.013671377, 0.01726313, -0.038217388, 0.006161347, 0.03463983, 0.0066937217, 0.027144, -0.02336769, -0.016780443, 0.003527868, 0.009440774, 0.009426577, -0.047274854, -0.017845193, 0.029273497, -0.0050611068, 0.03554842, 0.010555211, 0.021976419, -0.0005931539, 0.0001249971, -0.04542929, 0.033560887, 0.050824016, 0.028464288, 0.017206343, 0.02268625, 0.041340653, -0.0165391, 0.04233442, 0.03006851, -0.0049013942, -0.03231158, 0.009717609, -0.016255166, -0.041794945, 0.0018313682, 0.008993579, -0.06837108, -0.025142271, 0.015360778, -0.038756862, -0.037990242, 0.058319848, -0.020116657, -0.043526936, -0.005359236, -0.026136037, -0.008617368, 0.002047867, 0.0067540575, -0.013280968, -0.005305999, 0.018668598, 0.01649651, -0.021593109, -0.03296463, 0.0072757844, 0.016184185, -0.0009334299, 0.007218998, -0.0055083013, -0.014821305, 0.0027506014, 0.015474351, -0.0065836976, 0.03469662, -0.00085313007, -0.03515091, -0.0023637426, 0.01709277, 0.0067363116, 0.010363556, 0.019875314, 0.018881548, -0.03975063, -0.008979383, -0.0023814884, 0.02676069, -0.030693162, 0.028748222, 0.038047027, -0.013018331, -0.031743716, 0.005526047, 0.034355897, 0.019875314, -0.00041613937, 0.027924815, 0.022089992, -0.041993696, 0.014381209, -0.020428983, 0.02624961, -0.014508978, 0.00060335774, 0.019506201, 0.012606627, 0.019534593, 0.0318005, 0.0004622785, -0.05482747, -0.020329606, -0.014579962, 0.0025944381, 0.07581013, 0.0055828337, -0.027002033, 0.031743716, 0.0004711514, 0.06473674, -0.032226402, -0.012961544, 0.0123794805, 0.023765197, -0.056985363, -0.007928831, -0.018200109, 0.022416513, 0.033731245, -0.013941113, 0.061215967, 0.054060854, -0.025355222, 0.021451142, 0.009121349, 0.009632429, -0.052300468, -0.03367446, -0.011506387, -0.0065978942, 0.024418242, 0.02005987, 0.04667859, 0.017376702, 0.048893273, -0.014083079, 0.03444108, 0.017845193, -0.006622738, -0.047473606, -0.00958274, -0.03889883, -0.008766433, 0.0061897403, -0.03154496, 0.030466016, 0.06536139, -0.01920807, 0.0065730503, -0.022586875, 0.045031782, -0.039381515, 0.014665142, 0.041851733, -0.027200786, 0.06371458, -0.010562309, -0.0653046, -0.02323992, -0.05352138, -0.042306025, 0.01980433, -0.012102646, -0.024418242, 0.001565181, 0.015957037, -0.028762419, -0.012400776, -0.0062749204, -0.03299302, -0.055622485, 0.021564715, -0.004376118, 0.011953581, -0.0036112734, 0.0029103137, -0.011513486, 0.006420436, -0.00318005, -0.043867655, -0.011108881, 0.0029990429, 0.015062648, 0.029103138, -0.026348988, -0.021451142, 0.009398184, -0.0010541014, 0.00039240436, -0.020428983, -0.044974994, 0.012691807, 0.03188568, -0.0040637916, 0.031459782, -0.0067185657, -0.039892595, 0.0076945857, 0.010448736, 0.046195906, 0.012677611, 0.022061598, -0.06422566, 0.048694517, 0.0051888763, 0.02086908, 0.0116909435, 0.07348187, -0.020258622, -0.028038388, -0.02285661, 0.03211283, 0.06581568, 0.0067363116, 0.0011907442, 0.03023887, 0.017291524, 0.026604528, 0.025681743, 0.00027794385, -0.054146033, 0.0046032644, -0.009696313, -0.020727113, 0.04244799, -0.016595887, -0.02705882, -0.004177365, 0.012273006, -0.006640484, 0.035378058, -0.019279053, 0.035945926, 0.034157146, 0.03617307, 0.046451446, -0.052328862, -0.055878025, 0.005522498, -0.014217948, 0.003439139, -9.749107e-5, 0.028819205, 0.026093448, 0.009973149, 0.002684942, -0.039580267, 0.06876858, 0.0060406756, -0.029784577, 0.0033770287, 0.037592735, -0.003698228, -0.0065801484, 0.038984008, -0.015474351, -0.0059342007, -0.060875244, -0.010888833, 0.019875314, -0.0037053262, -0.015431761, 0.031204242, 0.012663414, -0.04193691, 0.007588111, -0.0330782, 0.033816427, 0.0057709394, 0.0098666735, -0.033134986, -0.06308992, -0.029756183, 0.021848649, -0.0008659958, -0.012635021, -0.02701623, -0.038359355, 0.010022837, 0.014388307, -0.01687982, 0.025355222, 0.041993696, -0.06797357, -0.022544283, 0.012791184, -0.022586875, 0.0047523296, 0.0081062885, -0.0077797654, -0.007218998, 0.018242698, 0.057837162, 0.015673105, -0.01632615, -0.03781988, -0.059739515, -0.074447244, -0.017234735, -0.015062648, -0.019193875, 0.010065427, -0.011584468, -0.050398115, 0.027144, 0.002038994, -0.023978146, -0.0010132861, -0.0065162634, -0.023126347, -0.0318005, 0.046877347, 0.023821983, -0.016851427, 0.043924443, -0.0131319035, -0.027726062, 0.023694213, -0.03324856, 0.013742359, -0.011158569, -0.004691994, -0.056218743, 0.0012927827, 0.024034932, 0.00010775038, -0.015701497, -0.03554842, -0.012485956, 0.044492308, -0.027882226, 0.021323372, 0.016808836, 0.012464661, -0.00017523993, 0.023509657, -0.0022448457, 0.06132954, 0.0065801484, 0.0026263807, -0.022501694, 0.0125072505, -0.021096226, -0.00038042592, 0.01764644, -0.046195906, 0.027456326, 0.016425528, -0.011889696, -0.027484719, -0.038785253, -0.017419292, 0.027144, -0.002527004, 0.013572, 0.0131957885, -0.02896117, 0.01051972, 0.033560887, -0.017461883, 0.01136442, -0.009625331, -0.029358678, 0.008816121, 0.04151101, 0.0023903614, -0.022672053, -0.0048375092, -0.051931355, 0.016382936, -0.030721556, -0.0061684456, 0.03676933, 0.029131532, -0.028861795, 0.002319378, -0.011847107, 0.021408552, 0.00873804, -0.012116843, 0.040517244, 0.017944569, 0.019832723, -0.0084825, -0.005568637, 0.0027896422, 0.034242325, 0.0020301214, -0.040091347, -0.013855933, 0.029358678, -0.00084869366, -0.03214122, -0.004248348, -0.00856768, -0.019435218, -0.018796368, 0.012095548, 0.02659033, -0.0352077, -0.03711005, -0.045571253, 0.030863522, -0.050114185, -0.02328251, -0.0044009625, -0.01700759, 0.039835807, 0.014991665, 0.00078125956, -0.02345287, -0.005007869, 0.030551197, -0.042504776, 0.0035172207, -0.017859388, 0.0030948701, -0.023651622, -0.025000306, 0.025227452, -0.03040923, 0.002078035, 0.011357322, -0.02272884, 0.03154496, -0.010427441, 0.023694213, 0.017575456, 0.0034959256, 0.07734336, -0.031062275, 0.003896981, -0.041340653, -0.015460155, 0.007999814, 0.017532866, 0.0064026904, 0.037507556, 0.02328251, -0.022998577, -0.043924443, 0.020670326, 0.0081701735, 0.004873001, -0.0045145354, -0.027200786, 0.00945497, -0.018938335, -0.0048162146, -0.02230294, -0.011357322, -0.020329606, 0.029841363, 0.0099944435, 0.03659897, -0.017972963, -0.012833774, 0.058263063, -0.028691435, 0.028819205, -0.01602802, -0.011130176, -0.01170514, -0.006242978, 0.008120486, -0.021138815, -0.009007776, 0.022373924, 0.0335041, 0.00720835, -0.010611998, -0.0014152288, 0.03787667, 0.036485396, 0.015502744, -0.021607306, -0.017987158, -0.01747608, -0.0039679646, 0.005479908, -0.028506879, -0.001751512, -0.003515446, 0.003875686, 0.023438673, -0.003100194, -0.04724646, 0.010065427, 0.014906485, -0.026093448, -0.043526936, 0.03498055, -0.026306396, 0.046025548, -0.017816799, 0.0010274827, -0.023978146, -0.012265908, -0.005501203, -0.021664092, 0.028932778, -0.020585146, 0.038529716, -0.02938707, -0.023935556, 0.0186544, 0.025270041, -0.013380345, 0.021649895, -0.031658534, -0.047019314, 0.017816799, 0.010356458, -0.014849698, -0.01967656, -0.037848275, -0.024986109, 0.02065613, 0.009639527, -0.0011703365, 0.029756183, -0.028066782, 0.0173909, -0.007687487, -0.03600271, -0.008134682, -0.0010931422, 0.019534593, 0.015673105, -0.030778343, -0.012535644, 0.0021046537, 0.019960493, 0.0040389476, -0.016184185, 0.016808836, 0.019733347, -0.0076732906, 0.012883462, -0.0055153994, -0.0046671494, -0.027470522, -0.035775565, -0.013089313, -0.044009622, 0.032794267, 0.014963272, 0.014906485, -0.0035775565, 0.0075029307, 0.03711005, 0.0065198126, 0.012578234, 0.009121349, -0.030295657, 0.022487497, 0.0055828337, -0.02052836, 0.009433676, 0.008028207, 0.0026991386, 0.019917903, 0.019690758, -0.016255166, -0.009405282, 0.019960493, 0.0008216313, -0.03711005, 0.0019626871, 0.0018420157, -0.008645762, -0.026363183, -0.037990242, -0.0009538376, -0.041880123, -0.03015369, 0.02607925, 0.022189368, -0.007006048, -0.0017586104, 0.0039679646, 0.02531263, 0.033191774, 0.002697364, -0.044038016, 0.021493731, -0.015587924, 0.055622485, -0.034071967, 0.030551197, 0.021153012, 0.0110237, -0.0064026904, 0.016013823, 0.027740259, -0.010562309, 0.014551569, 0.022828218, 0.0010585379, -0.03935312, 0.0038011537, 0.00945497, -0.01005123, -0.0012147011, -0.035434846, 0.006289117, 0.00013664278, -0.012202023, 0.00071604364, 0.0016272913, -0.021451142, -0.025624957, -0.010611998, -0.0007967871, 0.011477994, 0.020045673, -0.00882322, 0.0073467675, -0.0014090177, -0.046082333, -0.0018455648, 0.009383988, 0.024744766, 0.015346581, -0.009845379, -0.04258996, 0.003893432, -0.006047774, -0.012791184, -0.00096980884, -0.009206529, 6.35522e-5, -0.00033961056, 0.007595209, -0.020670326, 0.012656315, 0.028677238, -0.045230534, -0.025355222, 0.029784577, 0.004347725, -0.020727113, 0.0009564995, -0.0018331427, -0.042050485, -0.010888833, -0.042107273, -0.018767975, 0.012791184, -0.027229179, -0.0072757844, -0.011996171, 0.012684709, 0.018725384, -0.032737482, -0.026235415, 0.002960002, 0.0049510826, -0.02752731, 0.05550891, -0.010590702, 0.02065613, 0.030721556, 0.0060158316, -0.0030061412, 0.014920682, 0.008681253, -0.010462933, 0.020414786, -0.012698906, -0.037053265, -0.04716128, 0.0016343896, 0.040602427, 0.049290776, -0.017021786, 0.037734702, 0.026278004, 0.03092031, 0.01365718, -0.0034497865, 0.0054976535, 0.033702854, 0.024744766, 0.020514162, -0.007893339, -0.010470031, 0.0017257805, 0.022785626, -0.027981602, 0.015829267, 0.067632854, 0.023892967, 0.007091228, 0.010924324, 0.044549096, -0.0347818, -0.020585146, -0.040829573, 0.018725384, 0.020244427, -0.01327387, -0.027598292, 0.0004367689, -0.013146101, -0.014232144, -0.004486142, 0.014012096, 0.020854883, -0.008347631, -0.022146778, -0.02663292, 0.020173443, 0.04091475, -0.0031658534, -0.044038016, -0.0013513438, -0.044634275, -0.0019502651, -0.03884204, -0.009909264, 0.025326828, -0.06252206, 0.02281402, 0.0037230721, 0.021252388, 0.046053942, 0.01997469, -0.024475029, -0.010022837, 0.003270554, -0.0146367485, -0.030182084, 0.02052836, 0.021309175, -0.032851055, -0.015886053, 0.042476386, 0.0013832863, 0.027115606, -0.0071444656, 0.019321645, -0.00479137, -0.0076661925, 0.012883462, -0.0043157823, 0.0054941047, -0.014963272, -0.003829547, 0.040233314, -0.010221589, -0.022643661, -0.010981111, 0.025014501, 0.0073396694, 0.0141398655, 0.026860066, -0.013139002, -0.010001542, 0.018796368, 0.02697364, 0.012322694, 0.04040367, 0.020854883, 0.023864573, 0.01064749, -0.011186962, 0.02392136, 0.013422935, 0.03909758, 0.018810565, -0.05292512, -0.0098027885, 0.0649071, -0.005675112, 0.03126103, -0.0002081806, -0.044038016, -0.034242325, -0.0056431694, 0.03833096, 0.012287202, 0.011726435, -0.0075668157, -0.031743716, -0.014409603, -0.038813647, 0.0041702664, -0.01692241, 0.006069069, 0.0060406756, -0.020457376, -0.0041099307, 0.0080424035, -0.024844142, -0.0045961663, -0.04304425, -0.03608789, 0.012883462, 0.060420953, -0.013621688, -0.014260537, 0.007197703, -0.0360595, 0.036655758, 0.0133235585, -0.0065020667, -0.024971912, 0.008205665, 0.017305719, -0.055707663, -0.03446947, -0.004567773, -0.0076448973, -0.014849698, -0.014622552, -0.021948025, 0.0067079184, -0.005068205, 0.033702854, 0.01348682, -0.013472623, 0.0052527613, 0.013139002, -0.017944569, -0.05403246, -0.015971234, -0.043754082, 0.014146964, 0.010015738, -0.014359914, -0.0041241273, -0.009660822, -0.006480772, 0.043356575, -0.013472623, 0.021536322, 0.037422378, 0.03092031, 0.011598665, -0.01318869, 0.020684524, -0.012095548, -0.030636376, -0.034668226, 8.190803e-5, -0.0015447732, -0.0007799286, 0.008929694, 0.006047774, 0.02370841, -0.024702175, -0.034753405, 0.0042625447, 0.019051908, 0.012968642, 0.030977096, 0.0071693095, 0.018995121, -0.0081701735, 0.01009382, 0.009703412, 0.0009352045, -0.0119464835, -0.019279053, -0.033106595, -0.023481263, -0.0128763635, -0.0048055667, 0.025085485, -0.016212577, 0.015247204, 0.0013948211, -0.007389358, 0.012684709, -0.0035882038, -0.005334392, 0.0023743901, 0.014026293, -0.024120113, -0.0038224487, 0.019250661, 0.031573355, 0.020684524, 0.04795629, 0.032680694, -0.0034302662, -0.016567493, 0.036371823, -0.005746095, 0.024602799, 0.025923088, -0.038245782, -0.011087585, -0.028379109, 0.035576813, -0.0027239828, -0.01590025, 0.004358372, 0.020499967, -0.008709646, 0.009831182, -0.014949075, 0.0056964066, -0.001827819, 0.028237142, 0.0021490182, -0.050199363, 0.03040923, 0.010164803, -0.033702854, 0.043101035, -0.0018260444, -0.0035296427, 0.03495216, -0.025426205, 0.010171901, -0.010526818, -0.03083513, 0.025369419, -0.04091475, 0.0148922885, 0.006920868, 0.0020709366, -0.028577862, -0.0010966914, 0.030437622, -0.018853154, 0.0022164523, -0.006331707, 0.03889883, -0.028577862, 0.029017957, 0.028946975, 0.099263, 0.010285474, -0.009447873, -0.015233008, 0.0037301704, -0.025511384, -0.00061444886, 0.0020709366, -0.027073016, 0.013586196, -0.003109067, 0.00318005, -0.013586196, -0.038359355, -0.0013894973, 0.010732669, -0.02443244, 0.022132581, 0.014849698, -0.024900928, 0.016524903, 0.0018615361, 0.0074390457, -8.856271e-5, -0.039722234, -0.04795629, 0.04886488, 0.022757234, 0.021706682, 0.037393983, 0.0043441756, 0.0071906047, 0.005348589, 0.027868029, -0.016425528, -0.011669648, -0.0040850868, -0.0084257135, -0.0142676355, 0.006005184, -0.0062997644, -0.007879142, -0.009724707, 0.025000306, 0.0017115839, -0.016311953, 0.012642119, 0.028648844, -0.012358186, -0.032623906, -0.004322881, -0.042504776, -0.027498916, 0.0052279173, 0.031232635, -0.043952838, 0.007268686, -0.047303244, -0.009731805, 0.021124618, -0.0082979435, 0.015985431, -0.0057070544, -0.0330782, 0.003197796, -0.027456326, -0.03781988, -2.243182e-5, 0.0019751093, 0.007137367, 0.012109744, 0.035463236, -0.020940062, 0.009227824, -0.020358, -0.009490462, 0.004699092, -0.0015092816, 0.004851706, -0.010832046, 0.0040566935, -0.0019520397, 0.024205292, -0.0081701735, -0.035860743, -0.029756183, -0.010690079, -0.010782357, -0.040602427, 0.022899201, 0.011506387, -0.057808768, -0.005593481, -0.012315596, 0.01662428, -0.0037337197, 0.010413244, 0.013919818, 0.009902165, 0.0045074373, -0.0059058075, -0.0056005795, -0.032737482, 0.013117707, -0.0040637916, 0.02808098, 0.011392813, 0.01658169, -0.024290472, 0.0013034302, 0.012251711, 0.0032013452, -0.032339975, -0.018242698, 0.010349359, 0.0017861164, -0.013820441, 0.009128448, 0.02103944, -0.03256712, 0.031374604, -0.004873001, -0.014090178, -0.018072339, 0.03057959, -0.021067832, 0.0037727605, -0.0094194785, 0.017234735, -0.037905063, -0.0006863194, -0.024077523, 0.010739768, -0.010782357, -7.4421514e-5, -0.018597614, 0.0074106525, -0.033702854, -0.01149219, -0.0148922885, 0.00083050417, 0.010406146, 0.024673782, -0.005529596, -0.002175637, -0.008496696, 0.004457749, -0.0044471016, -0.026519347, -0.046195906, 0.00035691273, -0.0051924256, -0.052527614, -0.031686928, 0.021337569, -0.024105916, 0.008958087, 0.02921671, 0.0015412241, -0.009816986, -0.019704953, 0.02010246, -8.240713e-5, -0.026703903, 0.038359355, 0.03975063, -0.0006375184, -0.016553298, 0.0047700754, -0.007687487, 0.0011357322, -0.036031105, 0.025184862, 0.022388121, -0.034753405, -0.0077371756, 0.008851613, 0.031516567, -0.020826489, -0.002328251, -0.002179186, -0.04699092, 0.016382936, -0.0267181, 0.007687487, -0.0118825985, 0.046082333, -0.005423121, 0.0007439933, -0.0048055667, 0.03935312, 0.0064062392, -0.016638476, -0.013713966, -0.0062536253, 0.01594284, 0.0016157565, 0.026945246, 0.024233686, -0.025468795, -0.006047774, 0.013990801, 0.002599762, 0.024531815, -0.024276275, -0.030437622, -0.005114344, -0.0018810565, -0.027470522, -0.062351696, 0.007282883, -0.017972963, -0.00424125, 0.015573728, -0.019818528, 0.009731805, -0.03651379, 0.018271092, -0.009440774, 0.024091719, 0.039665446, 0.014473488, 0.00019875313, 0.011265044, 0.0051924256, -0.021124618, 0.010164803, 0.007687487, 0.0021153013, -0.008447008, 0.028435895, 0.022019008, 0.0035740072, -0.016226774, 0.014381209, 0.030466016, -0.04520214, 0.030182084, -0.02981297, 0.003057604, 0.028819205, 0.034043573, 0.02116721, 0.007701684, -0.03160175, 0.008539286, -0.016354544, -0.010129311, 0.0061471504, 0.0402901, -0.012713103, -0.019179678, 0.047473606, -0.028946975, -0.011776123, -0.021692485, -0.0133874435, -0.02116721, 0.022828218, -0.028024193, 0.004031849, -0.0010168352, 0.032084435, 0.020982653, -0.016127396, 0.0098027885, -0.032169614, -0.02569594, -0.0038579402, -0.019577185, -0.03864329, 0.042760316, -0.013181591, -0.0017444136, -0.003121489, -0.0013087539, -0.045400895, 0.06184062, -0.008986481, 0.03583235, 0.04287389, 0.00814178, 0.029103138, -2.4428224e-5, 0.02782544, -0.0055579892, 0.008021109, 0.018455649, -0.001211152, 0.009859575, -0.008880006, -0.02091167, 0.0020318958, 0.011030799, 0.008177272, 0.008965186, -0.004780723, 0.034100357, -0.009163939, 0.03523609, -0.0009990894, 0.015602121, 0.021749271, 0.0084895985, -0.006441731, 0.018100731, 0.0053734328, 0.024546012, -0.039012402, 0.026746493, 0.043015856, 0.0005785136, 0.01318869, 0.016638476, 0.016297758, -0.015474351, 0.008979383, 0.035576813, 0.0006401803, 0.011378617, -0.002841105, 0.008681253, 0.014650946, -0.009618232, 0.00029857334, 0.0005421347, 0.0073822592, -0.025965678, 0.020159246, 0.014516077, -0.02328251, 0.0054479656, 0.020570949, 0.014367012, -0.030352443, 0.0039573167, 0.0428455, 0.03736559, 0.008340533, -0.009355594, 0.00093875366, -0.035094123, -0.012677611, 0.010910127, 0.010079623, -0.007701684, 0.0049475334, 0.017859388, 0.0233251, -0.022501694, -0.03211283, -0.013181591, 0.0020035026, -0.014409603, -0.021692485, 0.019193875, 0.025937283, -0.024219489, -0.0013460201, -0.024631193, -0.013891424, 0.00020296777, -0.0025766925, 0.0041170293, 0.03498055, 0.020130854, -0.02646256, 0.022104189, 0.03367446, 0.030778343, 0.025724335, -0.020755505, 0.0070770313, -0.029614218, 0.0024844143, -0.024531815, -0.010718472, 0.013962408, 0.0073183742, -0.013209986, -0.008496696, -0.0028517526, -0.039324727, -0.042504776, -0.010498424, 0.043981228, -0.010462933, 0.027158197, -0.0068463357], "index": 0 } ], From 561002d734e439f6345bcd0cd209ac9ac6fc4638 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Thu, 23 Jan 2025 22:39:12 +0400 Subject: [PATCH 08/10] chore: add readme and license --- rig-postgres/LICENSE | 7 +++ rig-postgres/README.md | 127 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 rig-postgres/LICENSE create mode 100644 rig-postgres/README.md diff --git a/rig-postgres/LICENSE b/rig-postgres/LICENSE new file mode 100644 index 0000000000..878b5fbc3d --- /dev/null +++ b/rig-postgres/LICENSE @@ -0,0 +1,7 @@ +Copyright (c) 2024, Playgrounds Analytics Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rig-postgres/README.md b/rig-postgres/README.md new file mode 100644 index 0000000000..bae0c04c27 --- /dev/null +++ b/rig-postgres/README.md @@ -0,0 +1,127 @@ +
+ + + + Rig logo + + + + + + Postgres logo + +
+ +

+ +## Rig-postgres + +This companion crate implements a Rig vector store based on PostgreSQL. + +## Usage + +Add the companion crate to your `Cargo.toml`, along with the rig-core crate: + +```toml +[dependencies] +rig-core = "0.4.0" +rig-postgres = "0.1.0" +``` + +You can also run `cargo add rig-core rig-postgres` to add the most recent versions of the dependencies to your project. + +## PostgreSQL setup + +The crate utilizes [pgvector](https://github.com/pgvector/pgvector) extension, which is available for PostgreSQL version 13 and later. Use any of the [official](https://www.postgresql.org/download/) or alternative methods to install psql. + +You can install Postgres using Docker: + +```sh +docker pull pgvector/pgvector:pg17 + +docker run -e POSTGRES_USER=myuser \ + -e POSTGRES_PASSWORD=mypassword \ + -e POSTGRES_DB=mydatabase \ + --name my_postgres \ + -p 5432:5432 \ + -d ankane/pgvector +``` + +Now you can configure Postgres, the recommended way is using sqlx and migrations (you can find an example inside integration tests folder). + +Example sql: + +```sql +-- ensure PgVector extension is installed +CREATE EXTENSION IF NOT EXISTS vector; + +-- create table with embeddings using 1536 dimensions (based on OpenAI model text-embedding-3-small) +CREATE TABLE documents ( + id uuid DEFAULT gen_random_uuid(), -- we can have repeated entries + document jsonb NOT NULL, + embedded_text text NOT NULL, + embedding vector(1536) +); + +-- create index on embeddings +CREATE INDEX IF NOT EXISTS document_embeddings_idx ON documents +USING hnsw(embedding vector_cosine_ops); -- recommended for text embeddings + +``` + +You can change the table name and the number of dimensions but keep the same fields schema. + +You can use different indexes depending the type of distance method you want to use, check [PgVector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#querying). + +## Usage + +Declare the database URL: + +```sh +export DATABASE_URL="postgres://myuser:mypassword@localhost:5432/mydatabase" +``` + +Define the document you want to index, it has to implement `Embed`, `Serialize` and `Deserialize`. + +> Note: you can index different type of documents in the same table. + +Example: + +```rust +#[derive(Embed, Clone, Serialize, Deserialize, Debug)] +pub struct Product { + name: String, + category: String, + #[embed] + description: String, + price: f32 +} +``` + +Example usage + +```rust + // Create OpenAI client + let openai_client = rig::providers::openai::Client::from_env(); + let model = openai_client.embedding_model(rig::providers::openai::TEXT_EMBEDDING_3_SMALL); + + // init documents + let products: Vec = ...; + + let documents = EmbeddingsBuilder::new(model.clone()) + .documents(products) + .unwrap() + .build() + .await?; + + // Create your index + let vector_store = PostgresVectorStore::default(model, pool); + + // store documents + vector_store.insert_documents(documents).await?; + + // retrieve embeddings + let results = vector_store.top_n::("Which phones have more than 16Gb and support 5G", 50).await? + + ... + +``` From f91ae6320899d82438c212357b6951e812a45661 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Fri, 24 Jan 2025 12:14:00 +0400 Subject: [PATCH 09/10] chore: updates based on pr comments --- Cargo.lock | 2 - rig-postgres/Cargo.toml | 9 +-- rig-postgres/README.md | 7 +++ .../examples/vector_search_postgres.rs | 6 +- rig-postgres/src/lib.rs | 57 ++++++++++++++----- rig-postgres/tests/integration_tests.rs | 27 ++++++++- 6 files changed, 84 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38cbaf0e77..7a89855543 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6510,7 +6510,6 @@ version = "0.1.0" dependencies = [ "anyhow", "dotenvy", - "futures", "httpmock", "log", "pgvector", @@ -6520,7 +6519,6 @@ dependencies = [ "sqlx", "testcontainers", "tokio", - "tokio-stream", "tokio-test", "tracing", "tracing-subscriber", diff --git a/rig-postgres/Cargo.toml b/rig-postgres/Cargo.toml index 4e00ba2fb8..3ffbd18f47 100644 --- a/rig-postgres/Cargo.toml +++ b/rig-postgres/Cargo.toml @@ -4,11 +4,10 @@ version = "0.1.0" edition = "2021" description = "PostgreSQL-based vector store implementation for the rig framework" license = "MIT" +readme = "README.md" +repository = "https://github.com/0xPlaygrounds/rig" [dependencies] -futures = "0.3.31" -tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] } -tokio-stream = "0.1.6" rig-core = { path = "../rig-core", version = "0.6.0", features = ["derive"] } serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.133" @@ -26,8 +25,10 @@ uuid = { version = "1.11.0", features = ["v4", "serde"] } [dev-dependencies] anyhow = "1.0.94" log = "0.4.22" -tokio-test = "0.4.4" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tokio-test = "0.4.4" +tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] } + testcontainers = "0.23.1" httpmock = "0.7.0" diff --git a/rig-postgres/README.md b/rig-postgres/README.md index bae0c04c27..cd424c49b7 100644 --- a/rig-postgres/README.md +++ b/rig-postgres/README.md @@ -104,6 +104,13 @@ Example usage let openai_client = rig::providers::openai::Client::from_env(); let model = openai_client.embedding_model(rig::providers::openai::TEXT_EMBEDDING_3_SMALL); + // connect to Postgres + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL not set"); + let pool = PgPoolOptions::new() .connect(&database_url) .await?; + + // run migrations (optional but recommended) + sqlx::migrate!("./migrations").run(&pool).await?; + // init documents let products: Vec = ...; diff --git a/rig-postgres/examples/vector_search_postgres.rs b/rig-postgres/examples/vector_search_postgres.rs index fc9adf1750..ce4cbe1951 100644 --- a/rig-postgres/examples/vector_search_postgres.rs +++ b/rig-postgres/examples/vector_search_postgres.rs @@ -76,9 +76,11 @@ async fn main() -> Result<(), anyhow::Error> { .await .expect("Failed to create embeddings"); - // init vector store + // delete documents from table to have a clean start (optional, not recommended for production) sqlx::query("TRUNCATE documents").execute(&pool).await?; - let vector_store = PostgresVectorStore::default(model, pool); + + // init vector store + let vector_store = PostgresVectorStore::with_defaults(model, pool); vector_store.insert_documents(documents).await?; // query vector diff --git a/rig-postgres/src/lib.rs b/rig-postgres/src/lib.rs index 7dd2cb220d..43fadb8b26 100644 --- a/rig-postgres/src/lib.rs +++ b/rig-postgres/src/lib.rs @@ -55,6 +55,12 @@ pub struct SearchResult { distance: f64, } +#[derive(Debug, Deserialize, sqlx::FromRow)] +pub struct SearchResultOnlyId { + id: Uuid, + distance: f64, +} + impl SearchResult { pub fn into_result(self) -> Result<(f64, String, T), VectorStoreError> { let document: T = @@ -78,21 +84,29 @@ impl PostgresVectorStore { } } - pub fn default(model: Model, pg_pool: PgPool) -> Self { + pub fn with_defaults(model: Model, pg_pool: PgPool) -> Self { Self::new(model, pg_pool, None, PgVectorDistanceFunction::Cosine) } - fn search_query(&self) -> String { + fn search_query_full(&self) -> String { + self.search_query(true) + } + fn search_query_only_ids(&self) -> String { + self.search_query(false) + } + + fn search_query(&self, with_document: bool) -> String { + let document = if with_document { ", document" } else { "" }; format!( " - SELECT id, document, distance FROM ( \ - SELECT DISTINCT ON (id) id, document, embedding {} $1 as distance \ + SELECT id{}, distance FROM ( \ + SELECT DISTINCT ON (id) id{}, embedding {} $1 as distance \ FROM {} \ ORDER BY id, distance \ - ) \ + ) as d \ ORDER BY distance \ LIMIT $2", - self.distance_function, self.documents_table + document, document, self.distance_function, self.documents_table ) } @@ -147,7 +161,7 @@ impl VectorStoreIndex for PostgresVectorStore { .collect::>() .into(); - let rows: Vec = sqlx::query_as(self.search_query().as_str()) + let rows: Vec = sqlx::query_as(self.search_query_full().as_str()) .bind(embedded_query) .bind(n as i64) .fetch_all(&self.pg_pool) @@ -168,11 +182,28 @@ impl VectorStoreIndex for PostgresVectorStore { query: &str, n: usize, ) -> Result, VectorStoreError> { - self.top_n::(query, n).await.map(|result| { - result - .into_iter() - .map(|(distance, id, _)| (distance, id)) - .collect() - }) + let embedded_query: pgvector::Vector = self + .model + .embed_text(query) + .await? + .vec + .iter() + .map(|&x| x as f32) + .collect::>() + .into(); + + let rows: Vec = sqlx::query_as(self.search_query_only_ids().as_str()) + .bind(embedded_query) + .bind(n as i64) + .fetch_all(&self.pg_pool) + .await + .map_err(|e| VectorStoreError::DatastoreError(Box::new(e)))?; + + let rows: Vec<(f64, String)> = rows + .into_iter() + .map(|row| (row.distance, row.id.to_string())) + .collect(); + + Ok(rows) } } diff --git a/rig-postgres/tests/integration_tests.rs b/rig-postgres/tests/integration_tests.rs index c955079460..71aeb32790 100644 --- a/rig-postgres/tests/integration_tests.rs +++ b/rig-postgres/tests/integration_tests.rs @@ -75,7 +75,7 @@ async fn vector_search_test() { .expect("Failed to create embeddings"); // insert documents into vector store - let vector_store = PostgresVectorStore::default(model, pg_pool.clone()); + let vector_store = PostgresVectorStore::with_defaults(model, pg_pool.clone()); vector_store .insert_documents(documents) @@ -102,10 +102,31 @@ async fn vector_search_test() { results.len() ); - let (distance, id, doc) = results[0].clone(); - println!("Distance: {}, id: {}, document: {:?}", distance, id, doc); + let (distance, full_query_id, doc) = results[0].clone(); + println!( + "Distance: {}, id: {}, document: {:?}", + distance, full_query_id, doc + ); assert_eq!(doc.name, "glarb-glarb"); + + // search only ids + let results = vector_store + .top_n_ids("What does \"glarb-glarb\" mean?", 1) + .await + .expect("Failed to search for document ids"); + + assert_eq!( + results.len(), + 1, + "Expected one (id) result, got {}", + results.len() + ); + + let (distance, id) = results[0].clone(); + println!("Distance: {}, id: {}", distance, id); + + assert_eq!(id, full_query_id); } async fn start_container() -> ContainerAsync { From ca8c3aace3274521ac3c992c38a391d094a4e565 Mon Sep 17 00:00:00 2001 From: Carlos Verdes Date: Fri, 24 Jan 2025 22:34:41 +0400 Subject: [PATCH 10/10] fix: conflicts --- Cargo.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a89855543..053e331365 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5619,22 +5619,22 @@ dependencies = [ ] [[package]] -name = "pharos" -version = "0.5.3" +name = "pgvector" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +checksum = "e0e8871b6d7ca78348c6cd29b911b94851f3429f0cd403130ca17f26c1fb91a6" dependencies = [ - "futures", - "rustc_version", + "sqlx", ] [[package]] -name = "pgvector" -version = "0.4.0" +name = "pharos" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e8871b6d7ca78348c6cd29b911b94851f3429f0cd403130ca17f26c1fb91a6" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ - "sqlx", + "futures", + "rustc_version", ] [[package]]