diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 000000000..ff1a88b51 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,84 @@ +# feat(ingestion): implementar fundação do módulo de ingestão com TimescaleDB e dual-write Discord + +**Resolve:** #299 (Fase 1 — somente Discord) + +## Contexto + +Hoje cada módulo `integration-*` escreve diretamente no Postgres transacional usado por login, painéis, gamification e identity. Com ~3.4 GB de dados de atividade acumulados e novos provedores planejados (Instagram, WhatsApp), essa coabitação é insustentável. + +Este PR inicializa o módulo `ingestion` como futuro ponto único de entrada para dados de todos os provedores, apoiado por uma instância dedicada de TimescaleDB. O escopo é deliberadamente limitado ao **Discord** — Twitch, GitHub e Dev.to virão em PRs separados após o time alinhar onde os DTOs de cada provedor devem morar (`integration-*` ou `ingestion`). + +## O que mudou + +### Infraestrutura + +| Arquivo | Alteração | +| --------------------- | --------------------------------------------------------------------------------------------------- | +| `docker-compose.yml` | Adicionado serviço `he4rtbot-timescaledb` (TimescaleDB HA na porta `5436`) com volume e healthcheck | +| `config/database.php` | Adicionada conexão `timescaledb` apontando para a nova instância | +| `composer.json` | Registrado `he4rt/ingestion` como dependência do módulo | + +### Novo módulo: `app-modules/ingestion/` + +| Arquivo | Finalidade | +| --------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `database/migrations/…_create_timescaledb_tables.php` | Cria `raw_payloads` (cofre append-only), hypertable `messages` (particionada por `sent_at`) e hypertable `voice_messages` (particionada por `occurred_at`) | +| `Providers/IngestionServiceProvider.php` | Escuta o evento `discord.message.received` e registra o comando de backfill | +| `Listeners/ProcessRawDiscordMessage.php` | Listener na fila `ingestion` — salva o JSON original em `raw_payloads`, depois roda o Transform | +| `Actions/TransformDiscordMessage.php` | Usa o `DiscordMessageDTO::fromDump()` + `toDatabase()` existente para garantir paridade campo a campo com o insert legado no Postgres | +| `Models/RawPayload.php` | Model Eloquent apontando para a conexão `timescaledb` | +| `Models/Message.php` | Model Eloquent com override de PK composta (`id` + `sent_at`) exigida pelo TimescaleDB | +| `Console/Commands/BackfillPostgresToTimescaleCommand.php` | Cópia chunked e idempotente (`upsert`) do histórico de mensagens do Postgres → TimescaleDB | +| `tests/Feature/DualWriteAndBackfillTest.php` | Valida a corretude do backfill e o fluxo completo evento → raw_payload → hypertable | + +### Arquivos modificados + +| Arquivo | Alteração | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `MessageReceivedEvent.php` | Adicionado `event('discord.message.received', ['raw_payload' => …])` no topo do handler — persistência legada intacta (dual-write) | + +## Arquitetura + +``` +Discord Bot (WS) + │ + ├─ event('discord.message.received') ← NOVO (async, fila) + │ │ + │ ▼ + │ ProcessRawDiscordMessage (queue: ingestion) + │ │ + │ ├─ RawPayload::create() → TimescaleDB.raw_payloads + │ └─ TransformDiscordMessage → TimescaleDB.messages (hypertable) + │ + └─ Persistência legada (sync) → Postgres.messages ← INTOCADO +``` + +Ambos os caminhos rodam de forma independente. O Postgres legado continua sendo a fonte da verdade até o backfill atingir paridade (Decisão 10). + +## O que NÃO está neste PR + +- Ingestão de mensagens de voz (a tabela `voice_messages` foi criada para adiantar a estrutura, mas o listener e o ETL de voz virão na próxima fase). +- Ingestão de Twitch / GitHub / Dev.to (pendente discussão sobre ownership dos DTOs) +- Continuous aggregates / views materializadas +- Feature flag para cutover do dual-write +- Migração das queries do dashboard (`external_identity_id` → `external_account_id`) + +## Como testar + +```bash +# Suba o banco novo e rode as migrations +docker compose up -d he4rtbot-timescaledb +php artisan migrate --path=app-modules/ingestion/database/migrations --database=timescaledb + +# Inicie o worker da nova fila e o bot +php artisan queue:work --queue=ingestion,default +php artisan bot:boot + +# Em outro terminal, teste a idempotência do backfill +php artisan ingestion:backfill-postgres-timescale +``` + +## Notas de deploy + +> [!WARNING] +> Produção requer uma instância de TimescaleDB e workers escutando a fila `ingestion`. O dashboard e o sistema de XP continuam lendo do Postgres legado — nenhuma mudança de comportamento para o usuário final. diff --git a/app-modules/bot-discord/src/Events/MessageReceivedEvent.php b/app-modules/bot-discord/src/Events/MessageReceivedEvent.php index 211f12934..3aec2102e 100644 --- a/app-modules/bot-discord/src/Events/MessageReceivedEvent.php +++ b/app-modules/bot-discord/src/Events/MessageReceivedEvent.php @@ -33,6 +33,14 @@ public function handle(Message $message): void } try { + /** + * Dispatches the raw message payload to the ingestion queue for + * asynchronous processing (activity tracking, telemetry, moderation). + */ + event('discord.message.received', [ + 'raw_payload' => $message->getRawAttributes(), + ]); + $tenantProvider = resolve(ResolveDiscordTenant::class)->handle((string) $message->guild_id); // Activity tracking — records message for XP/gamification regardless of moderation outcome. diff --git a/app-modules/ingestion/composer.json b/app-modules/ingestion/composer.json new file mode 100644 index 000000000..b2bf75e42 --- /dev/null +++ b/app-modules/ingestion/composer.json @@ -0,0 +1,27 @@ +{ + "name": "he4rt/ingestion", + "description": "", + "type": "library", + "version": "1.0.0", + "license": "proprietary", + "autoload": { + "psr-4": { + "He4rt\\Ingestion\\": "src/", + "He4rt\\Ingestion\\Database\\Factories\\": "database/factories/", + "He4rt\\Ingestion\\Database\\Seeders\\": "database/seeders/" + } + }, + "autoload-dev": { + "psr-4": { + "He4rt\\Ingestion\\Tests\\": "tests/" + } + }, + "minimum-stability": "stable", + "extra": { + "laravel": { + "providers": [ + "He4rt\\Ingestion\\Providers\\IngestionServiceProvider" + ] + } + } +} diff --git a/app-modules/ingestion/database/factories/.gitkeep b/app-modules/ingestion/database/factories/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app-modules/ingestion/database/migrations/.gitkeep b/app-modules/ingestion/database/migrations/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app-modules/ingestion/database/migrations/2026_06_09_000000_create_timescaledb_tables.php b/app-modules/ingestion/database/migrations/2026_06_09_000000_create_timescaledb_tables.php new file mode 100644 index 000000000..39be90cdf --- /dev/null +++ b/app-modules/ingestion/database/migrations/2026_06_09_000000_create_timescaledb_tables.php @@ -0,0 +1,85 @@ +statement('CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;'); + + Schema::connection('timescaledb')->dropIfExists('voice_messages'); + Schema::connection('timescaledb')->dropIfExists('messages'); + Schema::connection('timescaledb')->dropIfExists('raw_payloads'); + + Schema::connection('timescaledb')->create('raw_payloads', function (Blueprint $table): void { + $table->uuid('id')->primary(); + $table->string('provider'); + $table->string('event_type'); + $table->jsonb('payload'); + $table->timestampsTz(); + }); + + Schema::connection('timescaledb')->create('messages', function (Blueprint $table): void { + $table->uuid('id'); + $table->uuid('tenant_id')->nullable(); + $table->string('external_identity_id'); + $table->string('provider_message_id')->nullable(); + $table->string('channel_id')->nullable(); + $table->text('content'); + $table->integer('obtained_experience')->default(0); + $table->jsonb('metadata')->nullable(); + $table->unsignedInteger('reactions_count')->default(0); + $table->unsignedInteger('reactions_total')->default(0); + $table->string('kind')->nullable(); + $table->smallInteger('raw_message_type')->nullable(); + $table->string('source_kind')->nullable(); + $table->boolean('is_pinned')->default(false); + $table->boolean('mentions_everyone')->default(false); + $table->smallInteger('mention_role_count')->default(0); + $table->timestampTz('edited_at')->nullable(); + $table->string('reply_to_provider_message_id')->nullable(); + + // Time column for partitioning + $table->timestampTz('sent_at'); + $table->timestampsTz(); + + // Composite primary key (required for Timescale hypertable) + $table->primary(['id', 'sent_at']); + }); + + DB::connection('timescaledb')->statement("SELECT create_hypertable('messages', 'sent_at');"); + + Schema::connection('timescaledb')->create('voice_messages', function (Blueprint $table): void { + $table->uuid('id'); + $table->uuid('tenant_id')->nullable(); + $table->string('external_identity_id'); + $table->string('channel_name'); + $table->string('channel_id')->nullable(); + $table->string('state'); + $table->integer('obtained_experience')->default(0); + $table->string('provider_message_id')->nullable(); + + $table->timestampTz('occurred_at'); + $table->timestampsTz(); + + $table->primary(['id', 'occurred_at']); + }); + + DB::connection('timescaledb')->statement("SELECT create_hypertable('voice_messages', 'occurred_at');"); + } + + public function down(): void + { + Schema::connection('timescaledb')->dropIfExists('voice_messages'); + Schema::connection('timescaledb')->dropIfExists('messages'); + Schema::connection('timescaledb')->dropIfExists('raw_payloads'); + } +}; diff --git a/app-modules/ingestion/database/seeders/.gitkeep b/app-modules/ingestion/database/seeders/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app-modules/ingestion/phpstan.ignore.neon b/app-modules/ingestion/phpstan.ignore.neon new file mode 100644 index 000000000..f51e71c3f --- /dev/null +++ b/app-modules/ingestion/phpstan.ignore.neon @@ -0,0 +1,2 @@ +parameters: + ignoreErrors: [] diff --git a/app-modules/ingestion/phpstan.neon b/app-modules/ingestion/phpstan.neon new file mode 100644 index 000000000..b577d0f29 --- /dev/null +++ b/app-modules/ingestion/phpstan.neon @@ -0,0 +1,6 @@ +includes: + - phpstan.ignore.neon + +parameters: + paths: + - src/ diff --git a/app-modules/ingestion/src/Actions/TransformDiscordMessage.php b/app-modules/ingestion/src/Actions/TransformDiscordMessage.php new file mode 100644 index 000000000..ec4dcd065 --- /dev/null +++ b/app-modules/ingestion/src/Actions/TransformDiscordMessage.php @@ -0,0 +1,45 @@ +payload; + + if (blank($data['id'] ?? null) || blank(data_get($data, 'author.id'))) { + return null; + } + + $dto = DiscordMessageDTO::fromDump($data); + + $tenantId = null; + if (filled($data['guild_id'])) { + $tenantId = DiscordGuild::query()->where('discord_guild_id', $data['guild_id'])->value('tenant_id'); + } + + $extraColumns = [ + 'tenant_id' => $tenantId, + 'external_identity_id' => $dto->authorDiscordId, + 'raw_message_type' => $data['type'] ?? null, + 'is_pinned' => $data['pinned'] ?? false, + 'mentions_everyone' => $data['mention_everyone'] ?? false, + 'mention_role_count' => count($data['mention_roles'] ?? []), + 'edited_at' => isset($data['edited_timestamp']) + ? Date::parse($data['edited_timestamp']) + : null, + 'reply_to_provider_message_id' => $data['message_reference']['message_id'] ?? null, + ]; + + return Message::query()->create($dto->toDatabase($extraColumns)); + } +} diff --git a/app-modules/ingestion/src/Console/Commands/BackfillPostgresToTimescaleCommand.php b/app-modules/ingestion/src/Console/Commands/BackfillPostgresToTimescaleCommand.php new file mode 100644 index 000000000..51f972886 --- /dev/null +++ b/app-modules/ingestion/src/Console/Commands/BackfillPostgresToTimescaleCommand.php @@ -0,0 +1,64 @@ +info('Starting message backfill from PostgreSQL to TimescaleDB...'); + + $chunkSize = (int) $this->option('chunk'); + $processed = 0; + + DB::connection('pgsql')->table('messages') + ->orderBy('id') + ->chunk($chunkSize, function ($messages) use (&$processed): void { + $payloads = []; + + foreach ($messages as $msg) { + $payloads[] = [ + 'id' => $msg->id, + 'tenant_id' => $msg->tenant_id, + 'external_identity_id' => $msg->external_identity_id, + 'provider_message_id' => $msg->provider_message_id, + 'channel_id' => $msg->channel_id, + 'content' => $msg->content, + 'obtained_experience' => $msg->obtained_experience, + 'metadata' => $msg->metadata, + 'reactions_count' => $msg->reactions_count, + 'reactions_total' => $msg->reactions_total, + 'kind' => $msg->kind, + 'raw_message_type' => $msg->raw_message_type, + 'source_kind' => $msg->source_kind, + 'is_pinned' => $msg->is_pinned, + 'mentions_everyone' => $msg->mentions_everyone, + 'mention_role_count' => $msg->mention_role_count, + 'edited_at' => $msg->edited_at, + 'reply_to_provider_message_id' => $msg->reply_to_provider_message_id, + 'sent_at' => $msg->sent_at, + 'created_at' => $msg->created_at, + 'updated_at' => $msg->updated_at, + ]; + } + + DB::connection('timescaledb')->table('messages')->upsert($payloads, ['id', 'sent_at']); + + $processed += count($messages); + $this->info(sprintf('Processed %d messages...', $processed)); + }); + + $this->info('Backfill completed successfully!'); + + return self::SUCCESS; + } +} diff --git a/app-modules/ingestion/src/Listeners/ProcessRawDiscordMessage.php b/app-modules/ingestion/src/Listeners/ProcessRawDiscordMessage.php new file mode 100644 index 000000000..28cffa40f --- /dev/null +++ b/app-modules/ingestion/src/Listeners/ProcessRawDiscordMessage.php @@ -0,0 +1,40 @@ +create([ + 'provider' => 'discord', + 'event_type' => 'message_create', + 'payload' => $rawPayload, + ]); + + try { + (new TransformDiscordMessage)->execute($record); + } catch (Throwable $throwable) { + Log::error('[Ingestion] Failed to transform Discord message', [ + 'raw_payload_id' => $record->id, + 'error' => $throwable->getMessage(), + ]); + } + } +} diff --git a/app-modules/ingestion/src/Models/Message.php b/app-modules/ingestion/src/Models/Message.php new file mode 100644 index 000000000..4981cf3b3 --- /dev/null +++ b/app-modules/ingestion/src/Models/Message.php @@ -0,0 +1,45 @@ +where($this->getKeyName(), '=', $this->getKeyForSaveQuery()); + + if (isset($this->sent_at)) { + $query->where('sent_at', '=', $this->sent_at); + } + + return $query; + } + + protected function casts(): array + { + return [ + 'metadata' => 'array', + 'is_pinned' => 'boolean', + 'mentions_everyone' => 'boolean', + 'sent_at' => 'datetime', + 'edited_at' => 'datetime', + ]; + } +} diff --git a/app-modules/ingestion/src/Models/RawPayload.php b/app-modules/ingestion/src/Models/RawPayload.php new file mode 100644 index 000000000..24017a6e6 --- /dev/null +++ b/app-modules/ingestion/src/Models/RawPayload.php @@ -0,0 +1,26 @@ + 'array', + ]; + } +} diff --git a/app-modules/ingestion/src/Providers/IngestionServiceProvider.php b/app-modules/ingestion/src/Providers/IngestionServiceProvider.php new file mode 100644 index 000000000..766798bb8 --- /dev/null +++ b/app-modules/ingestion/src/Providers/IngestionServiceProvider.php @@ -0,0 +1,25 @@ +commands([ + BackfillPostgresToTimescaleCommand::class, + ]); + } + + public function boot(): void + { + Event::listen('discord.message.received', ProcessRawDiscordMessage::class); + } +} diff --git a/app-modules/ingestion/tests/Feature/.gitkeep b/app-modules/ingestion/tests/Feature/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app-modules/ingestion/tests/Feature/DualWriteAndBackfillTest.php b/app-modules/ingestion/tests/Feature/DualWriteAndBackfillTest.php new file mode 100644 index 000000000..cd6dc26fa --- /dev/null +++ b/app-modules/ingestion/tests/Feature/DualWriteAndBackfillTest.php @@ -0,0 +1,80 @@ +table('messages')->truncate(); + + $tenant = Tenant::factory()->create(['slug' => 'test-tenant-'.Str::random(5)]); + + $identity = ExternalIdentity::factory()->create(['tenant_id' => $tenant->getKey()]); + + $postgresMessage = PostgresMessage::query()->create([ + 'id' => Str::uuid()->toString(), + 'tenant_id' => $tenant->getKey(), + 'external_identity_id' => $identity->id, + 'provider_message_id' => 'msg-123', + 'channel_id' => 'channel-123', + 'content' => 'Mensagem antiga do backfill', + 'obtained_experience' => 10, + 'reactions_count' => 0, + 'reactions_total' => 0, + 'is_pinned' => false, + 'mentions_everyone' => false, + 'mention_role_count' => 0, + 'sent_at' => now(), + ]); + + $this->artisan('ingestion:backfill-postgres-timescale') + ->assertExitCode(0); + + $this->assertDatabaseHas('messages', [ + 'id' => $postgresMessage->id, + 'content' => 'Mensagem antiga do backfill', + ], 'timescaledb'); + + expect(DB::connection('timescaledb')->table('raw_payloads')->count())->toBe(0); +}); + +test('ingestion listener saves raw payload and structures message in timescaledb', function (): void { + $tenant = Tenant::factory()->create(['slug' => 'discord-tenant-'.Str::random(5)]); + + DiscordGuild::factory()->create([ + 'discord_guild_id' => 'guild-999', + 'tenant_id' => $tenant->getKey(), + ]); + + $rawPayload = [ + 'id' => 'msg-new-999', + 'channel_id' => 'channel-999', + 'guild_id' => 'guild-999', + 'content' => 'Mensagem ao vivo!', + 'timestamp' => now()->toIso8601String(), + 'type' => 0, + 'author' => [ + 'id' => 'user-999', + 'username' => 'testuser', + 'discriminator' => '1234', + ], + ]; + + event('discord.message.received', ['raw_payload' => $rawPayload]); + + $this->assertDatabaseHas('raw_payloads', [ + 'provider' => 'discord', + 'event_type' => 'message_create', + ], 'timescaledb'); + + $this->assertDatabaseHas('messages', [ + 'provider_message_id' => 'msg-new-999', + 'content' => 'Mensagem ao vivo!', + 'tenant_id' => $tenant->getKey(), + ], 'timescaledb'); +}); diff --git a/app-modules/ingestion/tests/Unit/.gitkeep b/app-modules/ingestion/tests/Unit/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/composer.json b/composer.json index c1de77519..3ed64bc40 100644 --- a/composer.json +++ b/composer.json @@ -23,6 +23,7 @@ "he4rt/gamification": ">=1", "he4rt/he4rt-core": ">=1", "he4rt/identity": ">=1", + "he4rt/ingestion": ">=1", "he4rt/integration-devto": ">=1", "he4rt/integration-discord": ">=1", "he4rt/integration-github": ">=1", diff --git a/composer.lock b/composer.lock index cbfe501c4..13a949ba8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "36167e0f649a4d5090ff6da3d5f7b4de", + "content-hash": "139c8d2fe1db356d5ce34b48b8969fd3", "packages": [ { "name": "blade-ui-kit/blade-heroicons", @@ -3080,7 +3080,7 @@ "dist": { "type": "path", "url": "app-modules/activity", - "reference": "6f47fbd1735df0c7a357fcbbaeb62f75d4f87359" + "reference": "42abc699c506e672586e8b1f589d9f366b883737" }, "type": "library", "extra": { @@ -3116,7 +3116,7 @@ "dist": { "type": "path", "url": "app-modules/bot-discord", - "reference": "f0b37c36ff96f09d11efda00562148765c9e0844" + "reference": "6004355cb035839417a729d5e05d981f6fa29532" }, "type": "library", "extra": { @@ -3150,7 +3150,7 @@ "dist": { "type": "path", "url": "app-modules/community", - "reference": "fbfe830d8ba4cabdad48a59ab2c15770d097cdac" + "reference": "b3d3ec923c22eaf47e286985044f2daf2d318b39" }, "type": "library", "extra": { @@ -3186,7 +3186,7 @@ "dist": { "type": "path", "url": "app-modules/docs", - "reference": "65edac1e1f224a7e36a0f50104dbaa01dd55c909" + "reference": "e3f89b40bb03981923d0461c1b440b252820398d" }, "type": "library", "extra": { @@ -3220,7 +3220,7 @@ "dist": { "type": "path", "url": "app-modules/economy", - "reference": "42527108c63f20a2f26123e0159763d0be5df87b" + "reference": "f0ba33cf4c0343e55d8a64d6c69beceb69d8f768" }, "type": "library", "extra": { @@ -3256,7 +3256,7 @@ "dist": { "type": "path", "url": "app-modules/events", - "reference": "781fb24cba4bff8eed15b26d2964adf4f625a0a8" + "reference": "89dd2d40fa8d9ae6dee9e0b30ec67fa1f6f8f22c" }, "type": "library", "extra": { @@ -3292,7 +3292,7 @@ "dist": { "type": "path", "url": "app-modules/gamification", - "reference": "7ad03c485e9ebc39779174a027498bbff456f6ec" + "reference": "57af96e1d88a6bc9c5da9103cff1a242dcda6708" }, "type": "library", "extra": { @@ -3328,7 +3328,7 @@ "dist": { "type": "path", "url": "app-modules/he4rt", - "reference": "57b337ad69c6f649e1bb3b353e3037fd4459f617" + "reference": "291d9e96ad57589390c8fff90e14fa190f832434" }, "type": "library", "extra": { @@ -3357,7 +3357,7 @@ "dist": { "type": "path", "url": "app-modules/identity", - "reference": "649c67dbdd559cc5ab1a462ffe36cf25b8f0de9b" + "reference": "f3c2eee93f069fc96caa67d1584b3b2db6be9315" }, "type": "library", "extra": { @@ -3387,13 +3387,49 @@ "relative": true } }, + { + "name": "he4rt/ingestion", + "version": "1.0.0", + "dist": { + "type": "path", + "url": "app-modules/ingestion", + "reference": "0f8a59bd3ab155de1453e1b7ce600efca4ba506a" + }, + "type": "library", + "extra": { + "laravel": { + "providers": [ + "He4rt\\Ingestion\\Providers\\IngestionServiceProvider" + ] + } + }, + "autoload": { + "psr-4": { + "He4rt\\Ingestion\\": "src/", + "He4rt\\Ingestion\\Database\\Factories\\": "database/factories/", + "He4rt\\Ingestion\\Database\\Seeders\\": "database/seeders/" + } + }, + "autoload-dev": { + "psr-4": { + "He4rt\\Ingestion\\Tests\\": "tests/" + } + }, + "license": [ + "proprietary" + ], + "transport-options": { + "symlink": true, + "relative": true + } + }, { "name": "he4rt/integration-devto", "version": "1.0", "dist": { "type": "path", "url": "app-modules/integration-devto", - "reference": "b8c3771cfd5bd698ff7030a92090fa005403fda8" + "reference": "fdb4fb1cb5a5bb50423bd94310c907225001b3ec" }, "type": "library", "extra": { @@ -3429,7 +3465,7 @@ "dist": { "type": "path", "url": "app-modules/integration-discord", - "reference": "fa4e0c456d4ecfcd298bc70cdc7b98cd6fbaaa67" + "reference": "2c4764f485cd7051c853bc8830de3ad6c8c51585" }, "type": "library", "extra": { @@ -3465,7 +3501,7 @@ "dist": { "type": "path", "url": "app-modules/integration-github", - "reference": "37bb90924f560b468383db2a52b1b0017be05b22" + "reference": "d5811904452ce75a02d228e5de75f5b03e84f089" }, "type": "library", "extra": { @@ -3501,7 +3537,7 @@ "dist": { "type": "path", "url": "app-modules/integration-twitch", - "reference": "d84675d9e5ef87842e819789e9b8a3b9780bc651" + "reference": "bee16880c187fd3188837368aff33567e96494d2" }, "type": "library", "extra": { @@ -3537,7 +3573,7 @@ "dist": { "type": "path", "url": "app-modules/moderation", - "reference": "fd1acef105618c98bb5271f60fc94b4653f6e0a6" + "reference": "b635db1e9c9629b041cb3f69bb26140a67906d8a" }, "type": "library", "extra": { @@ -3574,7 +3610,7 @@ "dist": { "type": "path", "url": "app-modules/panel-admin", - "reference": "24bad248d7ef6e839a58fbc1cb4d64dee2200f9f" + "reference": "1ba694deda9c5175df8b8846079a186339cd5818" }, "type": "library", "extra": { @@ -3610,7 +3646,7 @@ "dist": { "type": "path", "url": "app-modules/panel-app", - "reference": "02eeddd07e58ed1e1524ca1c83dc5b0955205c57" + "reference": "67a01a3fc1531c0a6cc37bdfb527c3a3af3de29f" }, "require": { "filament/filament": "^5.6", @@ -3651,7 +3687,7 @@ "dist": { "type": "path", "url": "app-modules/portal", - "reference": "289230f39053d3339ee643d7b32cf3be89f6d94e" + "reference": "d040051bf854b93165122628c8c11267d4dd8c05" }, "type": "library", "extra": { @@ -3687,7 +3723,7 @@ "dist": { "type": "path", "url": "app-modules/profile", - "reference": "5de02564b117b5734231584cdf338c7cefa6fc9b" + "reference": "b4b6bf011a401021942cb44791f6bef6d1caac34" }, "type": "library", "extra": { diff --git a/config/database.php b/config/database.php index 54bf5d63a..5124eafb8 100644 --- a/config/database.php +++ b/config/database.php @@ -101,6 +101,21 @@ 'sslmode' => 'prefer', ], + 'timescaledb' => [ + 'driver' => 'pgsql', + 'url' => env('DB_TIMESCALEDB_URL'), + 'host' => env('DB_TIMESCALEDB_HOST', '127.0.0.1'), + 'port' => env('DB_TIMESCALEDB_PORT', '5436'), + 'database' => env('DB_TIMESCALEDB_DATABASE', 'ingestion_he4rtbot'), + 'username' => env('DB_TIMESCALEDB_USERNAME', 'postgres'), + 'password' => env('DB_TIMESCALEDB_PASSWORD', 'postgres'), + 'charset' => env('DB_CHARSET', 'utf8'), + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'public', + 'sslmode' => 'prefer', + ], + 'sqlsrv' => [ 'driver' => 'sqlsrv', 'url' => env('DB_URL'), diff --git a/docker-compose.yml b/docker-compose.yml index 544b8c7d8..2fe4fc1f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,30 @@ services: timeout: 10s start_period: 15s + he4rtbot-timescaledb: + image: timescale/timescaledb-ha:pg16 + container_name: he4rtbot-timescaledb + restart: no + hostname: dev-ts + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: ingestion_he4rtbot + TZ: Etc/UTC + PGTZ: Etc/UTC + ports: + - '5436:5432' + volumes: + - he4rtbot-timescaledb:/home/postgres/pgdata/data + networks: + - dev-he4rtbot + healthcheck: + test: ['CMD-SHELL', 'pg_isready --quiet --username=postgres --dbname=ingestion_he4rtbot || exit 1'] + interval: 10s + retries: 5 + timeout: 10s + start_period: 15s + he4rtbot-redis: build: dockerfile: docker/redis.Dockerfile @@ -68,6 +92,8 @@ services: volumes: he4rtbot-db: name: he4rtbot-db + he4rtbot-timescaledb: + name: he4rtbot-timescaledb networks: dev-he4rtbot: diff --git a/package-lock.json b/package-lock.json index 710f058a1..9c0973c6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,6 @@ "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" @@ -48,7 +47,6 @@ "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -162,7 +160,6 @@ "integrity": "sha512-x9l65fCE/pgoET6RQowgdgG8Xmzs44z6j6Hhg3coINCyCw9JBGJ5ZzMR2XHAM2jmAdbJAIgqB6cUn4/3W3XLTA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "linguist-languages": "^8.0.0", "php-parser": "^3.2.5" @@ -1644,7 +1641,6 @@ "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -1689,7 +1685,6 @@ "integrity": "sha512-UKii4RjY05SNt/WQi6/NcOn/LsT0/ILLXsxygjbRg5/YZelsSu5jTqorYHPDGq4nZy5q5hpCu+XdGZ1xaJEQgw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=20.19" }, @@ -1945,8 +1940,7 @@ "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.0.tgz", "integrity": "sha512-y6nxMGB1nMW9R6k96e5gdIFzcfL/gTJRNaqGes1YvkLnPVXzWgbqFF2yLC0T8G774n24cx3Pe8XrKoniCOAH+Q==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tapable": { "version": "2.3.3", @@ -2029,7 +2023,6 @@ "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4",