feat: initialize ingest-api with Koa and telemetry handling

- Create ingest-api with Koa framework for handling device telemetry.
- Implement telemetry ingestion with MongoDB, Redis, and PostgreSQL.
- Add API endpoints for telemetry and health checks.
- Set up testing with Vitest for unit and end-to-end tests.
- Configure Prettier and ESLint for code formatting and linting.
- Create Docker Compose setup for PostgreSQL, MongoDB, and Redis services.
- Establish contracts package for shared Zod schemas and TypeScript types.
- Implement React frontend with device status and greeting functionalities.
- Add VS Code setup documentation for onboarding developers.
This commit is contained in:
marco.chan committed 2026-09-03 10:13:02 +08:00
1 parent 534fa6dc51
commit 7923ba1b8d
70 files changed
+3085 -3434

No files matched your search

+4
View File
@@ -0,0 +1,4 @@
{
"singleQuote": true,
"trailingComma": "all"
}
+3
View File
@@ -0,0 +1,3 @@
# ingest-api
Koa device ingest. `POST /telemetry` writes raw events to Mongo, a TTL key to Redis, and upserts `device_current_state` in Postgres. Dual-write is a demo shortcut — production should derive Redis/Postgres from a queue.
+10
View File
@@ -0,0 +1,10 @@
{
"$schema": "https://raw.githubusercontent.com/oxc-project/oxc/main/crates/oxc_linter/src/rules.rs",
"rules": {
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-floating-promises": "warn"
},
"env": {
"node": true
}
}
+40
View File
@@ -0,0 +1,40 @@
{
"name": "ingest-api",
"version": "0.0.1",
"description": "Koa device ingest API",
"private": true,
"license": "UNLICENSED",
"type": "module",
"scripts": {
"build": "tsc -p tsconfig.build.json",
"start": "tsx src/main.ts",
"start:dev": "tsx watch src/main.ts",
"start:debug": "tsx watch --inspect=9230 src/main.ts",
"lint": "oxlint src/ test/",
"test": "vitest run",
"test:e2e": "vitest run --config ./vitest.config.e2e.ts"
},
"dependencies": {
"@koa/cors": "^5.0.0",
"@koa/router": "^12",
"contracts": "workspace:*",
"ioredis": "^5",
"koa": "^2",
"koa-body": "^6",
"mongodb": "^6",
"pg": "^8"
},
"devDependencies": {
"@types/koa": "^2",
"@types/koa__cors": "^5.0.1",
"@types/koa__router": "^12",
"@types/node": "^24.0.0",
"@types/pg": "^8",
"@types/supertest": "^7.0.0",
"oxlint": "^1.58.0",
"supertest": "^7.0.0",
"tsx": "^4",
"typescript": "^6.0.2",
"vitest": "^4.1.2"
}
}
+38
View File
@@ -0,0 +1,38 @@
import request from 'supertest';
import type { TelemetryIngest } from 'contracts';
import { createApp } from './app.js';
import { createMemoryWriter } from './stores.js';
describe('ingest-api', () => {
const writes: TelemetryIngest[] = [];
const app = createApp(createMemoryWriter(writes));
it('GET /', async () => {
await request(app.callback()).get('/').expect(200).expect('ingest-api');
});
it('accepts telemetry', async () => {
writes.length = 0;
const body = {
deviceId: 'dev-1',
sentAt: new Date().toISOString(),
payload: { temp: 21 },
};
await request(app.callback())
.post('/telemetry')
.send(body)
.expect(200)
.expect({
accepted: true,
writes: { mongo: true, redis: true, postgres: true },
});
expect(writes).toHaveLength(1);
expect(writes[0]?.deviceId).toBe('dev-1');
});
it('rejects invalid telemetry', async () => {
await request(app.callback()).post('/telemetry').send({ deviceId: '' }).expect(400);
});
});
+39
View File
@@ -0,0 +1,39 @@
import cors from '@koa/cors';
import Router from '@koa/router';
import { telemetryIngestSchema } from 'contracts';
import Koa from 'koa';
import { koaBody } from 'koa-body';
import { createTelemetryWriter, type TelemetryWriter } from './stores.js';
export function createApp(writer: TelemetryWriter = createTelemetryWriter()): Koa {
const app = new Koa();
const router = new Router();
app.use(cors());
app.use(koaBody());
router.get('/', (ctx) => {
ctx.body = 'ingest-api';
});
router.get('/health', (ctx) => {
ctx.body = { ok: true };
});
router.post('/telemetry', async (ctx) => {
const parsed = telemetryIngestSchema.safeParse(ctx.request.body);
if (!parsed.success) {
ctx.status = 400;
ctx.body = { error: parsed.error.flatten() };
return;
}
const writes = await writer.write(parsed.data);
ctx.body = { accepted: true, writes };
});
app.use(router.routes());
app.use(router.allowedMethods());
return app;
}
+7
View File
@@ -0,0 +1,7 @@
export const env = {
databaseUrl: process.env.DATABASE_URL ?? 'postgres://app:app@127.0.0.1:5432/app',
mongoUrl: process.env.MONGO_URL ?? 'mongodb://127.0.0.1:27017/app',
redisUrl: process.env.REDIS_URL ?? 'redis://127.0.0.1:6379',
port: Number(process.env.INGEST_PORT ?? process.env.PORT ?? 3001),
host: process.env.HOST ?? '0.0.0.0',
};
+22
View File
@@ -0,0 +1,22 @@
import { createApp } from './app.js';
import { env } from './env.js';
import { createTelemetryWriter } from './stores.js';
const writer = createTelemetryWriter();
const app = createApp(writer);
const server = app.listen(env.port, env.host, () => {
console.log(`ingest-api listening on http://${env.host}:${env.port}`);
});
async function shutdown() {
server.close();
await writer.close();
}
process.on('SIGINT', () => {
void shutdown().then(() => process.exit(0));
});
process.on('SIGTERM', () => {
void shutdown().then(() => process.exit(0));
});
+100
View File
@@ -0,0 +1,100 @@
import { PRESENCE_TTL_SECONDS, type TelemetryIngest } from 'contracts';
import Redis from 'ioredis';
import { MongoClient } from 'mongodb';
import pg from 'pg';
import { env } from './env.js';
export type WriteReport = {
mongo: boolean;
redis: boolean;
postgres: boolean;
};
export type TelemetryWriter = {
write(event: TelemetryIngest): Promise<WriteReport>;
close(): Promise<void>;
};
function redisKey(deviceId: string): string {
return `device:${deviceId}`;
}
export function createTelemetryWriter(): TelemetryWriter {
const mongo = new MongoClient(env.mongoUrl);
const redis = new Redis(env.redisUrl, { maxRetriesPerRequest: 1, lazyConnect: true });
const pool = new pg.Pool({ connectionString: env.databaseUrl });
let mongoReady: Promise<MongoClient> | null = null;
function mongoClient(): Promise<MongoClient> {
mongoReady ??= mongo.connect();
return mongoReady;
}
return {
async write(event: TelemetryIngest): Promise<WriteReport> {
const report: WriteReport = { mongo: false, redis: false, postgres: false };
const lastSeen = new Date(event.sentAt);
try {
const client = await mongoClient();
await client.db().collection('telemetry').insertOne({
deviceId: event.deviceId,
sentAt: lastSeen,
payload: event.payload,
ingestedAt: new Date(),
});
report.mongo = true;
} catch (error) {
console.error('mongo ingest write failed', error);
}
try {
if (redis.status === 'wait') {
await redis.connect();
}
await redis.set(
redisKey(event.deviceId),
JSON.stringify({ lastSeen: lastSeen.toISOString(), payload: event.payload }),
'EX',
PRESENCE_TTL_SECONDS,
);
report.redis = true;
} catch (error) {
console.error('redis ingest write failed', error);
}
try {
await pool.query(
`INSERT INTO device_current_state (device_id, payload, last_seen, updated_at)
VALUES ($1, $2::jsonb, $3, NOW())
ON CONFLICT (device_id) DO UPDATE SET
payload = EXCLUDED.payload,
last_seen = EXCLUDED.last_seen,
updated_at = NOW()`,
[event.deviceId, JSON.stringify(event.payload), lastSeen],
);
report.postgres = true;
} catch (error) {
console.error('postgres ingest write failed', error);
}
return report;
},
async close() {
await Promise.allSettled([mongo.close(), redis.quit(), pool.end()]);
},
};
}
export function createMemoryWriter(writes: TelemetryIngest[] = []): TelemetryWriter {
return {
async write(event: TelemetryIngest) {
writes.push(event);
return { mongo: true, redis: true, postgres: true };
},
async close() {
return;
},
};
}
+18
View File
@@ -0,0 +1,18 @@
import request from 'supertest';
import { createApp } from '../src/app.js';
import { createMemoryWriter } from '../src/stores.js';
describe('ingest-api (e2e)', () => {
const app = createApp(createMemoryWriter());
it('POST /telemetry', async () => {
await request(app.callback())
.post('/telemetry')
.send({
deviceId: 'dev-1',
sentAt: new Date().toISOString(),
payload: { temp: 21 },
})
.expect(200);
});
});
+9
View File
@@ -0,0 +1,9 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"types": ["node"]
},
"include": ["src"],
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
}
+20
View File
@@ -0,0 +1,20 @@
{
"compilerOptions": {
"module": "nodenext",
"moduleResolution": "nodenext",
"resolvePackageJsonExports": true,
"esModuleInterop": true,
"isolatedModules": true,
"declaration": true,
"removeComments": true,
"allowSyntheticDefaultImports": true,
"target": "ES2023",
"sourceMap": true,
"outDir": "./dist",
"incremental": true,
"skipLibCheck": true,
"strict": true,
"types": ["vitest/globals", "node"]
},
"include": ["src", "test"]
}
+9
View File
@@ -0,0 +1,9 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
root: './',
include: ['**/*.e2e-spec.ts'],
},
});
+9
View File
@@ -0,0 +1,9 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
root: './',
include: ['**/*.spec.ts'],
},
});