feat(processor): ✨ Introduce AggregationService for data aggregation and SchemaGuardService for schema validation with module registration
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
parent
490724424a
commit
0c0cfc0b69
3 changed files with 41 additions and 1 deletions
|
|
@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
|
|||
import { HealthModule } from './health/health.module';
|
||||
import { ProcessorsModule } from './processors/processors.module';
|
||||
import { AggregatedMetric } from './entities/aggregated-metric.entity';
|
||||
import { SchemaGuardService } from './schema-guard.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
|
|
@ -47,5 +48,6 @@ import { AggregatedMetric } from './entities/aggregated-metric.entity';
|
|||
HealthModule,
|
||||
ProcessorsModule,
|
||||
],
|
||||
providers: [SchemaGuardService],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ export class AggregationService implements OnModuleDestroy {
|
|||
}
|
||||
|
||||
async processEvent(event: ProcessableEvent): Promise<void> {
|
||||
const { eventType, timestamp, sessionId, userId, properties } = event;
|
||||
const { eventType, timestamp, userId, properties } = event;
|
||||
|
||||
const hourBucket = this.getTimeBucket(timestamp, TimeGranularity.HOUR);
|
||||
const dayBucket = this.getTimeBucket(timestamp, TimeGranularity.DAY);
|
||||
|
|
|
|||
38
services/processor/src/schema-guard.service.ts
Normal file
38
services/processor/src/schema-guard.service.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
/**
|
||||
* Ensures DDL that the entity decorators cannot express exists before the
|
||||
* processor starts draining the queue.
|
||||
*
|
||||
* The aggregation upsert (aggregation.service.ts) relies on
|
||||
* `ON CONFLICT ("metricType","granularity","timestamp","dimension","dimensionValue")`,
|
||||
* which requires a unique index over exactly those columns that treats NULLs
|
||||
* as equal — global metrics key on NULL dimension/dimensionValue, so a plain
|
||||
* UNIQUE constraint (NULLs distinct) never conflicts and every upsert fails.
|
||||
*
|
||||
* Prod runs `synchronize: false` with no migration runner, so the `@Unique`
|
||||
* decorator on AggregatedMetric is never applied there — and even where it
|
||||
* is (dev sync), it lacks NULLS NOT DISTINCT. The 2026-05-16 → 2026-06-07
|
||||
* outage was exactly this: a fresh table without the index, every
|
||||
* aggregation failing for three weeks. This guard makes the fix survive
|
||||
* fresh deploys and new per-provider databases.
|
||||
*
|
||||
* Requires PostgreSQL 15+ (NULLS NOT DISTINCT).
|
||||
*/
|
||||
@Injectable()
|
||||
export class SchemaGuardService implements OnModuleInit {
|
||||
private readonly logger = new Logger(SchemaGuardService.name);
|
||||
|
||||
constructor(@InjectDataSource() private readonly dataSource: DataSource) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.dataSource.query(`
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_aggregated_metrics_dedup
|
||||
ON aggregated_metrics ("metricType", "granularity", "timestamp", "dimension", "dimensionValue")
|
||||
NULLS NOT DISTINCT
|
||||
`);
|
||||
this.logger.log('uq_aggregated_metrics_dedup ensured (NULLS NOT DISTINCT)');
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue