feat: initialize ingest-api with Koa and telemetry handling
- Create ingest-api with Koa framework for handling device telemetry. - Implement telemetry ingestion with MongoDB, Redis, and PostgreSQL. - Add API endpoints for telemetry and health checks. - Set up testing with Vitest for unit and end-to-end tests. - Configure Prettier and ESLint for code formatting and linting. - Create Docker Compose setup for PostgreSQL, MongoDB, and Redis services. - Establish contracts package for shared Zod schemas and TypeScript types. - Implement React frontend with device status and greeting functionalities. - Add VS Code setup documentation for onboarding developers.
This commit is contained in:
1 parent
534fa6dc51
commit
7923ba1b8d
70 files changed
+3085
-3434
No files matched your search
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# ingest-api
|
||||
|
||||
Koa device ingest. `POST /telemetry` writes raw events to Mongo, a TTL key to Redis, and upserts `device_current_state` in Postgres. Dual-write is a demo shortcut — production should derive Redis/Postgres from a queue.
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/oxc-project/oxc/main/crates/oxc_linter/src/rules.rs",
|
||||
"rules": {
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-floating-promises": "warn"
|
||||
},
|
||||
"env": {
|
||||
"node": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"name": "ingest-api",
|
||||
"version": "0.0.1",
|
||||
"description": "Koa device ingest API",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"start": "tsx src/main.ts",
|
||||
"start:dev": "tsx watch src/main.ts",
|
||||
"start:debug": "tsx watch --inspect=9230 src/main.ts",
|
||||
"lint": "oxlint src/ test/",
|
||||
"test": "vitest run",
|
||||
"test:e2e": "vitest run --config ./vitest.config.e2e.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@koa/cors": "^5.0.0",
|
||||
"@koa/router": "^12",
|
||||
"contracts": "workspace:*",
|
||||
"ioredis": "^5",
|
||||
"koa": "^2",
|
||||
"koa-body": "^6",
|
||||
"mongodb": "^6",
|
||||
"pg": "^8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/koa": "^2",
|
||||
"@types/koa__cors": "^5.0.1",
|
||||
"@types/koa__router": "^12",
|
||||
"@types/node": "^24.0.0",
|
||||
"@types/pg": "^8",
|
||||
"@types/supertest": "^7.0.0",
|
||||
"oxlint": "^1.58.0",
|
||||
"supertest": "^7.0.0",
|
||||
"tsx": "^4",
|
||||
"typescript": "^6.0.2",
|
||||
"vitest": "^4.1.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import request from 'supertest';
|
||||
import type { TelemetryIngest } from 'contracts';
|
||||
import { createApp } from './app.js';
|
||||
import { createMemoryWriter } from './stores.js';
|
||||
|
||||
describe('ingest-api', () => {
|
||||
const writes: TelemetryIngest[] = [];
|
||||
const app = createApp(createMemoryWriter(writes));
|
||||
|
||||
it('GET /', async () => {
|
||||
await request(app.callback()).get('/').expect(200).expect('ingest-api');
|
||||
});
|
||||
|
||||
it('accepts telemetry', async () => {
|
||||
writes.length = 0;
|
||||
const body = {
|
||||
deviceId: 'dev-1',
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: { temp: 21 },
|
||||
};
|
||||
|
||||
await request(app.callback())
|
||||
.post('/telemetry')
|
||||
.send(body)
|
||||
.expect(200)
|
||||
.expect({
|
||||
accepted: true,
|
||||
writes: { mongo: true, redis: true, postgres: true },
|
||||
});
|
||||
|
||||
expect(writes).toHaveLength(1);
|
||||
expect(writes[0]?.deviceId).toBe('dev-1');
|
||||
});
|
||||
|
||||
it('rejects invalid telemetry', async () => {
|
||||
await request(app.callback()).post('/telemetry').send({ deviceId: '' }).expect(400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import cors from '@koa/cors';
|
||||
import Router from '@koa/router';
|
||||
import { telemetryIngestSchema } from 'contracts';
|
||||
import Koa from 'koa';
|
||||
import { koaBody } from 'koa-body';
|
||||
import { createTelemetryWriter, type TelemetryWriter } from './stores.js';
|
||||
|
||||
export function createApp(writer: TelemetryWriter = createTelemetryWriter()): Koa {
|
||||
const app = new Koa();
|
||||
const router = new Router();
|
||||
|
||||
app.use(cors());
|
||||
app.use(koaBody());
|
||||
|
||||
router.get('/', (ctx) => {
|
||||
ctx.body = 'ingest-api';
|
||||
});
|
||||
|
||||
router.get('/health', (ctx) => {
|
||||
ctx.body = { ok: true };
|
||||
});
|
||||
|
||||
router.post('/telemetry', async (ctx) => {
|
||||
const parsed = telemetryIngestSchema.safeParse(ctx.request.body);
|
||||
if (!parsed.success) {
|
||||
ctx.status = 400;
|
||||
ctx.body = { error: parsed.error.flatten() };
|
||||
return;
|
||||
}
|
||||
|
||||
const writes = await writer.write(parsed.data);
|
||||
ctx.body = { accepted: true, writes };
|
||||
});
|
||||
|
||||
app.use(router.routes());
|
||||
app.use(router.allowedMethods());
|
||||
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export const env = {
|
||||
databaseUrl: process.env.DATABASE_URL ?? 'postgres://app:app@127.0.0.1:5432/app',
|
||||
mongoUrl: process.env.MONGO_URL ?? 'mongodb://127.0.0.1:27017/app',
|
||||
redisUrl: process.env.REDIS_URL ?? 'redis://127.0.0.1:6379',
|
||||
port: Number(process.env.INGEST_PORT ?? process.env.PORT ?? 3001),
|
||||
host: process.env.HOST ?? '0.0.0.0',
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
import { createApp } from './app.js';
|
||||
import { env } from './env.js';
|
||||
import { createTelemetryWriter } from './stores.js';
|
||||
|
||||
const writer = createTelemetryWriter();
|
||||
const app = createApp(writer);
|
||||
|
||||
const server = app.listen(env.port, env.host, () => {
|
||||
console.log(`ingest-api listening on http://${env.host}:${env.port}`);
|
||||
});
|
||||
|
||||
async function shutdown() {
|
||||
server.close();
|
||||
await writer.close();
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
void shutdown().then(() => process.exit(0));
|
||||
});
|
||||
process.on('SIGTERM', () => {
|
||||
void shutdown().then(() => process.exit(0));
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { PRESENCE_TTL_SECONDS, type TelemetryIngest } from 'contracts';
|
||||
import Redis from 'ioredis';
|
||||
import { MongoClient } from 'mongodb';
|
||||
import pg from 'pg';
|
||||
import { env } from './env.js';
|
||||
|
||||
export type WriteReport = {
|
||||
mongo: boolean;
|
||||
redis: boolean;
|
||||
postgres: boolean;
|
||||
};
|
||||
|
||||
export type TelemetryWriter = {
|
||||
write(event: TelemetryIngest): Promise<WriteReport>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
function redisKey(deviceId: string): string {
|
||||
return `device:${deviceId}`;
|
||||
}
|
||||
|
||||
export function createTelemetryWriter(): TelemetryWriter {
|
||||
const mongo = new MongoClient(env.mongoUrl);
|
||||
const redis = new Redis(env.redisUrl, { maxRetriesPerRequest: 1, lazyConnect: true });
|
||||
const pool = new pg.Pool({ connectionString: env.databaseUrl });
|
||||
let mongoReady: Promise<MongoClient> | null = null;
|
||||
|
||||
function mongoClient(): Promise<MongoClient> {
|
||||
mongoReady ??= mongo.connect();
|
||||
return mongoReady;
|
||||
}
|
||||
|
||||
return {
|
||||
async write(event: TelemetryIngest): Promise<WriteReport> {
|
||||
const report: WriteReport = { mongo: false, redis: false, postgres: false };
|
||||
const lastSeen = new Date(event.sentAt);
|
||||
|
||||
try {
|
||||
const client = await mongoClient();
|
||||
await client.db().collection('telemetry').insertOne({
|
||||
deviceId: event.deviceId,
|
||||
sentAt: lastSeen,
|
||||
payload: event.payload,
|
||||
ingestedAt: new Date(),
|
||||
});
|
||||
report.mongo = true;
|
||||
} catch (error) {
|
||||
console.error('mongo ingest write failed', error);
|
||||
}
|
||||
|
||||
try {
|
||||
if (redis.status === 'wait') {
|
||||
await redis.connect();
|
||||
}
|
||||
await redis.set(
|
||||
redisKey(event.deviceId),
|
||||
JSON.stringify({ lastSeen: lastSeen.toISOString(), payload: event.payload }),
|
||||
'EX',
|
||||
PRESENCE_TTL_SECONDS,
|
||||
);
|
||||
report.redis = true;
|
||||
} catch (error) {
|
||||
console.error('redis ingest write failed', error);
|
||||
}
|
||||
|
||||
try {
|
||||
await pool.query(
|
||||
`INSERT INTO device_current_state (device_id, payload, last_seen, updated_at)
|
||||
VALUES ($1, $2::jsonb, $3, NOW())
|
||||
ON CONFLICT (device_id) DO UPDATE SET
|
||||
payload = EXCLUDED.payload,
|
||||
last_seen = EXCLUDED.last_seen,
|
||||
updated_at = NOW()`,
|
||||
[event.deviceId, JSON.stringify(event.payload), lastSeen],
|
||||
);
|
||||
report.postgres = true;
|
||||
} catch (error) {
|
||||
console.error('postgres ingest write failed', error);
|
||||
}
|
||||
|
||||
return report;
|
||||
},
|
||||
|
||||
async close() {
|
||||
await Promise.allSettled([mongo.close(), redis.quit(), pool.end()]);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createMemoryWriter(writes: TelemetryIngest[] = []): TelemetryWriter {
|
||||
return {
|
||||
async write(event: TelemetryIngest) {
|
||||
writes.push(event);
|
||||
return { mongo: true, redis: true, postgres: true };
|
||||
},
|
||||
async close() {
|
||||
return;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import request from 'supertest';
|
||||
import { createApp } from '../src/app.js';
|
||||
import { createMemoryWriter } from '../src/stores.js';
|
||||
|
||||
describe('ingest-api (e2e)', () => {
|
||||
const app = createApp(createMemoryWriter());
|
||||
|
||||
it('POST /telemetry', async () => {
|
||||
await request(app.callback())
|
||||
.post('/telemetry')
|
||||
.send({
|
||||
deviceId: 'dev-1',
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: { temp: 21 },
|
||||
})
|
||||
.expect(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"resolvePackageJsonExports": true,
|
||||
"esModuleInterop": true,
|
||||
"isolatedModules": true,
|
||||
"declaration": true,
|
||||
"removeComments": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"target": "ES2023",
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"incremental": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"types": ["vitest/globals", "node"]
|
||||
},
|
||||
"include": ["src", "test"]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
root: './',
|
||||
include: ['**/*.e2e-spec.ts'],
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
root: './',
|
||||
include: ['**/*.spec.ts'],
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user