feat: add pglite support (#174)

Author: luoling8192

config/config.example.yaml

@@ -1,6 +1,9 @@
 # Database settings
 database:
-  # You can use url or separate fields
+  # Database type: postgres, pglite
+  type: postgres
+  # PostgreSQL configuration (used when type: postgres)
+  # Can use URL or separate field configuration
   # url: postgres://postgres:postgres@localhost:5432/tg_search
   host: localhost
   port: 5433

cspell.config.yaml

@@ -19,6 +19,7 @@ words:
   - ofetch
   - ollama
   - onnotice
+  - pglite
   - pgvector
   - pinia
   - reka

docker-compose.yml

@@ -9,10 +9,13 @@ services:
       POSTGRES_PASSWORD: '123456'
     volumes:
       - ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql
-      - ./.postgres/data:/var/lib/postgresql/data
+      - pgvector_data:/var/lib/postgresql/data
     healthcheck:
       test: [CMD-SHELL, pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER]
       interval: 10s
       timeout: 5s
       retries: 5
     restart: unless-stopped
+
+volumes:
+  pgvector_data:

packages/common/src/composable/config.ts

@@ -6,23 +6,28 @@ import { dirname, join, resolve } from 'node:path'
 import { cwd } from 'node:process'
 import { findWorkspaceDir } from '@pnpm/find-workspace-dir'
 import defu from 'defu'
+import path from 'path-browserify-esm'
 import { safeParse } from 'valibot'
 import { parse, stringify } from 'yaml'
 
-import { configSchema } from '../helper/config-schema'
+import { configSchema, DatabaseType } from '../helper/config-schema'
 import { generateDefaultConfig } from '../helper/default-config'
 import { useLogger } from '../helper/logger'
 
 let config: Config
 const logger = useLogger('common:config')
 
-export async function useConfigPath(): Promise<string> {
+async function getWorkspacePath() {
   const workspaceDir = await findWorkspaceDir(cwd())
   if (!workspaceDir) {
     throw new Error('Failed to find workspace directory')
   }
 
-  const configPath = resolve(workspaceDir, 'config', 'config.yaml')
+  return workspaceDir
+}
+
+export async function useConfigPath(): Promise<string> {
+  const configPath = resolve(await getWorkspacePath(), 'config', 'config.yaml')
 
   logger.withFields({ configPath }).log('Config path')
 
@@ -34,12 +39,15 @@ export async function useConfigPath(): Promise<string> {
   return configPath
 }
 
-export async function useAssetsPath(): Promise<string> {
-  const workspaceDir = await findWorkspaceDir(cwd())
-  if (!workspaceDir) {
-    throw new Error('Failed to find workspace directory')
-  }
+export async function getDrizzlePath(): Promise<string> {
+  const workspaceDir = await getWorkspacePath()
+  const drizzlePath = resolve(workspaceDir, 'drizzle')
+  logger.withFields({ drizzlePath }).log('Drizzle migrations path')
+  return drizzlePath
+}
 
+export async function useAssetsPath(): Promise<string> {
+  const workspaceDir = await getWorkspacePath()
   const assetsPath = resolve(workspaceDir, 'assets')
 
   logger.withFields({ assetsPath }).log('Assets path')
@@ -77,6 +85,21 @@ export function getDatabaseDSN(config: Config): string {
   return database.url || `postgres://${database.user}:${database.password}@${database.host}:${database.port}/${database.database}`
 }
 
+export function getDatabaseFilePath(config: Config): string {
+  const { database, path: configPath } = config
+
+  let extension = ''
+  switch (database.type) {
+    case DatabaseType.PGLITE:
+      extension = '.pglite'
+      break
+    default:
+      return ''
+  }
+
+  return path.join(configPath.storage, `db${extension}`)
+}
+
 export function resolveStoragePath(path: string): string {
   if (path.startsWith('~')) {
     path = path.replace('~', homedir())

packages/common/src/helper/config-schema.ts

@@ -18,9 +18,14 @@ export enum EmbeddingDimension {
   DIMENSION_768 = 768,
 }
 
+export enum DatabaseType {
+  POSTGRES = 'postgres',
+  PGLITE = 'pglite',
+}
+
 export const proxyConfigSchema = object({
-  ip: string(),
-  port: number(),
+  ip: optional(string(), ''),
+  port: optional(number(), 0),
   MTProxy: optional(boolean()),
   secret: optional(string()),
   socksType: optional(enumType(SocksType)),
@@ -30,57 +35,58 @@ export const proxyConfigSchema = object({
 })
 
 export const databaseConfigSchema = object({
-  host: string(),
-  port: number(),
-  user: string(),
-  password: string(),
-  database: string(),
-
+  type: optional(enumType(DatabaseType), DatabaseType.POSTGRES),
+  host: optional(string(), 'localhost'),
+  port: optional(number(), 5432),
+  user: optional(string(), 'postgres'),
+  password: optional(string(), 'postgres'),
+  database: optional(string(), 'postgres'),
   url: optional(string()),
 })
 
 export const messageConfigSchema = object({
-  export: object({
-    batchSize: number(),
-    concurrent: number(),
-    retryTimes: number(),
-    maxTakeoutRetries: number(),
-  }),
-  batch: object({
-    size: number(),
-  }),
+  export: optional(object({
+    batchSize: optional(number(), 200),
+    concurrent: optional(number(), 3),
+    retryTimes: optional(number(), 3),
+    maxTakeoutRetries: optional(number(), 3),
+  }), {}),
+  batch: optional(object({
+    size: optional(number(), 100),
+  }), {}),
 })
 
 export const pathConfigSchema = object({
-  storage: string(),
-  dict: string(),
+  storage: optional(string(), '~/.telegram-search'),
+  dict: optional(string(), ''),
+  assets: optional(string(), ''),
 })
 
 export const telegramConfigSchema = object({
-  apiId: string(),
-  apiHash: string(),
-  phoneNumber: string(),
+  apiId: optional(string(), ''),
+  apiHash: optional(string(), ''),
+  phoneNumber: optional(string(), ''),
   proxy: optional(proxyConfigSchema),
 })
 
 export const embeddingConfigSchema = object({
-  provider: enumType(EmbeddingProvider),
-  model: string(),
-  dimension: optional(enumType(EmbeddingDimension)),
-  apiKey: optional(string()),
-  apiBase: optional(string()),
+  provider: optional(enumType(EmbeddingProvider), EmbeddingProvider.OPENAI),
+  model: optional(string(), 'text-embedding-3-small'),
+  dimension: optional(enumType(EmbeddingDimension), EmbeddingDimension.DIMENSION_1536),
+  apiKey: optional(string(), ''),
+  apiBase: optional(string(), ''),
 })
 
 export const apiConfigSchema = object({
-  telegram: telegramConfigSchema,
-  embedding: embeddingConfigSchema,
+  telegram: optional(telegramConfigSchema, {}),
+  embedding: optional(embeddingConfigSchema, {}),
 })
 
 export const configSchema = object({
-  database: databaseConfigSchema,
-  message: messageConfigSchema,
-  path: pathConfigSchema,
-  api: apiConfigSchema,
+  database: optional(databaseConfigSchema, {}),
+  message: optional(messageConfigSchema, {}),
+  path: optional(pathConfigSchema, {}),
+  api: optional(apiConfigSchema, {}),
 })
 
 export type Config = InferOutput<typeof configSchema>

packages/common/src/helper/default-config.ts

@@ -7,81 +7,15 @@ import { configSchema } from './config-schema'
 
 export function generateDefaultConfig(paths?: { storagePath?: string, assetsPath?: string }): Config {
   const defaultConfig = safeParse(configSchema, {
-    // Database settings
-    database: {
-      // Default database connection settings
-      host: 'localhost',
-      port: 5432,
-      user: 'postgres',
-      password: 'postgres',
-      database: 'postgres',
-    },
-
-    // Message settings
-    message: {
-      // Export settings
-      export: {
-        // Number of messages to fetch in each request
-        batchSize: 200,
-        // Number of concurrent requests
-        concurrent: 3,
-        // Number of retry attempts
-        retryTimes: 3,
-        // Number of retry attempts for takeout session (0 means infinite retries)
-        maxTakeoutRetries: 3,
-      },
-      // Database batch settings
-      batch: {
-        // Number of messages to save in each batch
-        size: 100,
-      },
-    },
-
-    // Path settings
     path: {
       storage: paths?.storagePath ?? '~/.telegram-search',
       assets: paths?.assetsPath ?? '',
       dict: path.join(paths?.assetsPath ?? '', 'dict.txt.big'),
     },
-
-    // API settings
-    api: {
-      // Telegram API settings
-      telegram: {
-        // These values should be provided in config.yaml
-        apiId: '',
-        apiHash: '',
-        phoneNumber: '',
-        // Optional proxy settings - will be used if provided
-        // proxy: {
-        //   ip: '',            // Proxy host (IP or hostname)
-        //   port: 0,           // Proxy port
-        //   MTProxy: false,    // Whether it's an MTProxy or a normal Socks proxy
-        //   secret: '',        // If using MTProxy, provide a secret
-        //   socksType: 5,      // If using Socks, choose 4 or 5
-        //   timeout: 2,        // Timeout (in seconds) for connection
-        //   username: '',      // Optional username for proxy auth
-        //   password: '',      // Optional password for proxy auth
-        // }
-      },
-      // OpenAI API settings
-      embedding: {
-        // Embedding provider
-        provider: 'openai',
-        // Embedding model
-        model: 'text-embedding-3-small',
-        // API key should be provided in config.yaml
-        apiKey: '',
-        // Optional API base URL
-        apiBase: '',
-        // Embedding dimension
-        dimension: 1536,
-      },
-    },
   })
 
   if (!defaultConfig.success) {
-    throw new Error('Failed to generate default config')
+    throw new Error('Failed to generate default config', { cause: defaultConfig.issues })
   }
 
   return defaultConfig.output

packages/core/package.json

@@ -20,6 +20,7 @@
     "lint:fix": "eslint src --fix"
   },
   "dependencies": {
+    "@electric-sql/pglite": "^0.3.1",
     "@tg-search/common": "workspace:*",
     "@xsai-ext/providers-cloud": "catalog:",
     "@xsai/embed": "catalog:",
@@ -33,7 +34,8 @@
     "postgres": "^3.4.5",
     "telegram": "^2.26.22",
     "unbuild": "^3.5.0",
-    "valibot": "^1.1.0"
+    "valibot": "^1.1.0",
+    "vectorlite": "^0.2.0"
   },
   "devDependencies": {
     "@types/node": "^22.15.18",