nune/Sharkey
1
0
Fork 0
mirror of https://activitypub.software/TransFem-org/Sharkey synced 2024-09-29 11:25:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

merge: Draft: Add /metrics endpoint (!613)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/613
This commit is contained in:
4censord 2024-09-23 20:12:19 +00:00
parent c224ed031f 1f4671e15e
commit ea3c0e0bea
11 changed files with 139 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
.gitlab-ci.yml
View file

@ -3,6 +3,7 @@ stages:
- deploy
testCommit:
allow_failure: true
stage: test
image: node:iron
services:
@ -35,6 +36,31 @@ testCommit:
- merge_requests
- stable
Build Container for mr:
stage: deploy
parallel:
matrix:
- ARCH: amd64
- ARCH: arm64
tags:
- ${ARCH}
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
before_script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
script:
- >-
/kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-latest
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
only:
- merge_requests
getImageTag:
stage: deploy
image: ubuntu:latest

1
packages/backend/package.json
View file

@ -155,6 +155,7 @@
"pg": "8.12.0",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"prom-client": "^15.1.3",
"promise-limit": "2.7.0",
"proxy-addr": "^2.0.7",
"pug": "3.0.3",

5
packages/backend/src/boot/master.ts
View file

@ -12,6 +12,7 @@ import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { collectDefaultMetrics, AggregatorRegistry } from 'prom-client';
import Logger from '@/logger.js';
import { loadConfig } from '@/config.js';
import type { Config } from '@/config.js';
@ -94,6 +95,10 @@ export async function masterMain() {
});
}
// initialize prom-client in the master
const aggregatorRegistry = new AggregatorRegistry();
collectDefaultMetrics();
if (envOption.disableClustering) {
if (envOption.onlyServer) {
await server();

6
packages/backend/src/boot/worker.ts
View file

@ -6,10 +6,10 @@
import cluster from 'node:cluster';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { collectDefaultMetrics, AggregatorRegistry } from 'prom-client';
import { envOption } from '@/env.js';
import { loadConfig } from '@/config.js';
import { jobQueue, server } from './common.js';
/**
* Init worker process
*/
@ -34,6 +34,10 @@ export async function workerMain() {
});
}
// initialize prom-client in the worker
const aggregatorRegistry = new AggregatorRegistry();
collectDefaultMetrics();
if (envOption.onlyServer) {
await server();
} else if (envOption.onlyQueue) {

7
packages/backend/src/misc/metrics.ts Normal file
View file

@ -0,0 +1,7 @@
import { register, Counter, Gauge } from 'prom-client'
export const jobs_metrics = new Counter({
name: "sharkey_jobs",
help: "Metrics about jobs",
labelNames: ['job', 'status', 'reason'] as const,
});

32
packages/backend/src/queue/QueueProcessorService.ts
View file

@ -6,6 +6,7 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Bull from 'bullmq';
import * as Sentry from '@sentry/node';
import { register, Gauge, Histogram } from 'prom-client';
import type { Config } from '@/config.js';
import { DI } from '@/di-symbols.js';
import type Logger from '@/logger.js';
@ -142,6 +143,19 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
}
const histogram = new Histogram({
name: 'sharkey_queue_timings',
help: 'Timings of jobs per queue',
labelNames: ['queue', 'job'] as const,
});
function runProcessorWithTimer(queue: String, processor: (job: Bull.Job) => void, job: Bull.Job) {
const end = histogram.startTimer();
processor(job);
end({queue, job: job.name});
}
//#region system
{
const processer = (job: Bull.Job) => {
@ -160,7 +174,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: System: ' + job.name }, () => processer(job));
} else {
return processer(job);
return runProcessorWithTimer('system', processer, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.SYSTEM),
@ -225,7 +239,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: DB: ' + job.name }, () => processer(job));
} else {
return processer(job);
return runProcessorWithTimer('db', processer, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.DB),
@ -257,7 +271,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Deliver' }, () => this.deliverProcessorService.process(job));
} else {
return this.deliverProcessorService.process(job);
return runProcessorWithTimer('deliver', this.deliverProcessorService.process, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.DELIVER),
@ -297,7 +311,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Inbox' }, () => this.inboxProcessorService.process(job));
} else {
return this.inboxProcessorService.process(job);
return runProcessorWithTimer('inbox', this.inboxProcessorService.process, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.INBOX),
@ -337,7 +351,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: UserWebhookDeliver' }, () => this.userWebhookDeliverProcessorService.process(job));
} else {
return this.userWebhookDeliverProcessorService.process(job);
return runProcessorWithTimer('userwebhookdeliver', this.userWebhookDeliverProcessorService.process, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.USER_WEBHOOK_DELIVER),
@ -377,7 +391,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: SystemWebhookDeliver' }, () => this.systemWebhookDeliverProcessorService.process(job));
} else {
return this.systemWebhookDeliverProcessorService.process(job);
return runProcessorWithTimer('systemwebhookdeliver', this.systemWebhookDeliverProcessorService.process, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.SYSTEM_WEBHOOK_DELIVER),
@ -427,7 +441,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Relationship: ' + job.name }, () => processer(job));
} else {
return processer(job);
return runProcessorWithTimer('relationship', processer, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.RELATIONSHIP),
@ -472,7 +486,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: ObjectStorage: ' + job.name }, () => processer(job));
} else {
return processer(job);
return runProcessorWithTimer('objectstorage', processer, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE),
@ -505,7 +519,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job));
} else {
return this.endedPollNotificationProcessorService.process(job);
return runProcessorWithTimer('endedpollnotification', this.endedPollNotificationProcessorService.process, job);
}
}, {
...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),

10
packages/backend/src/queue/processors/DeliverProcessorService.ts
View file

@ -24,6 +24,8 @@ import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type { DeliverJobData } from '../types.js';
import { jobs_metrics } from '@/misc/metrics.js'
@Injectable()
export class DeliverProcessorService {
private logger: Logger;
@ -55,6 +57,7 @@ export class DeliverProcessorService {
// ブロックしてたら中断
const meta = await this.metaService.fetch();
if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.toPuny(host))) {
jobs_metrics.inc({ job: 'delivery', status: 'skipped', reason: 'blocked' });
return 'skip (blocked)';
}
@ -69,6 +72,7 @@ export class DeliverProcessorService {
this.suspendedHostsCache.set(suspendedHosts);
}
if (suspendedHosts.map(x => x.host).includes(this.utilityService.toPuny(host))) {
jobs_metrics.inc({ job: 'delivery', status: 'skipped', reason: 'suspended' });
return 'skip (suspended)';
}
@ -92,7 +96,7 @@ export class DeliverProcessorService {
this.instanceChart.requestSent(i.host, true);
}
});
jobs_metrics.inc({ job: 'delivery', status: 'success', reason: '' });
return 'Success';
} catch (res) {
// Update stats
@ -135,15 +139,19 @@ export class DeliverProcessorService {
suspensionState: 'goneSuspended',
});
});
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: 'gone' });
throw new Bull.UnrecoverableError(`${host} is gone`);
}
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: res.statusCode });
throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
}
// 5xx etc.
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: 'remote_error' });
throw new Error(`${res.statusCode} ${res.statusMessage}`);
} else {
// DNS error, socket error, timeout ...
jobs_metrics.inc({ job: 'delivery', status: 'retrying', reason: 'remote_error' });
throw res;
}
}

33
packages/backend/src/server/MetricsServerService.ts Normal file
View file

@ -0,0 +1,33 @@
import { Inject, Injectable } from '@nestjs/common';
import {
AggregatorRegistry,
collectDefaultMetrics,
} from 'prom-client';
import { bindThis } from '@/decorators.js';
import type { FastifyInstance, FastifyPluginOptions } from 'fastify';
@Injectable()
export class MetricsServerService {
private register: AggregatorRegistry;
constructor(
) {
this.register = new AggregatorRegistry()
collectDefaultMetrics({
register: this.register,
});
}
@bindThis
public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) {
fastify.get('/', async (request, reply) => {
reply
.code(200)
.type(this.register.contentType)
.send(
await this.register.clusterMetrics(),
);
});
done();
}
}

2
packages/backend/src/server/ServerModule.ts
View file

@ -9,6 +9,7 @@ import { CoreModule } from '@/core/CoreModule.js';
import { ApiCallService } from './api/ApiCallService.js';
import { FileServerService } from './FileServerService.js';
import { HealthServerService } from './HealthServerService.js';
import { MetricsServerService } from './MetricsServerService.js';
import { NodeinfoServerService } from './NodeinfoServerService.js';
import { ServerService } from './ServerService.js';
import { WellKnownServerService } from './WellKnownServerService.js';
@ -60,6 +61,7 @@ import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js
ClientLoggerService,
FeedService,
HealthServerService,
MetricsServerService,
UrlPreviewService,
ActivityPubServerService,
FileServerService,

4
packages/backend/src/server/ServerService.ts
View file

@ -29,6 +29,7 @@ import { StreamingApiServerService } from './api/StreamingApiServerService.js';
import { WellKnownServerService } from './WellKnownServerService.js';
import { FileServerService } from './FileServerService.js';
import { HealthServerService } from './HealthServerService.js';
import { MetricsServerService } from './MetricsServerService.js';
import { ClientServerService } from './web/ClientServerService.js';
import { OpenApiServerService } from './api/openapi/OpenApiServerService.js';
import { MastodonApiServerService } from './api/mastodon/MastodonApiServerService.js';
@ -65,6 +66,7 @@ export class ServerService implements OnApplicationShutdown {
private nodeinfoServerService: NodeinfoServerService,
private fileServerService: FileServerService,
private healthServerService: HealthServerService,
private metricsServerService: MetricsServerService,
private clientServerService: ClientServerService,
private globalEventService: GlobalEventService,
private loggerService: LoggerService,
@ -114,6 +116,8 @@ export class ServerService implements OnApplicationShutdown {
fastify.register(this.oauth2ProviderService.createServer, { prefix: '/oauth' });
fastify.register(this.healthServerService.createServer, { prefix: '/healthz' });
fastify.register(this.metricsServerService.createServer, { prefix: '/metrics' });
fastify.get<{ Params: { path: string }; Querystring: { static?: any; badge?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
const path = request.params.path;

24
pnpm-lock.yaml
View file

@ -355,6 +355,9 @@ importers:
probe-image-size:
specifier: 7.2.3
version: 7.2.3
prom-client:
specifier: ^15.1.3
version: 15.1.3
promise-limit:
specifier: 2.7.0
version: 2.7.0
@ -6006,6 +6009,9 @@ packages:
resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
engines: {node: '>=8'}
bintrees@1.0.2:
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
bl@4.1.0:
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
@ -9968,6 +9974,10 @@ packages:
resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
engines: {node: '>= 0.6.0'}
prom-client@15.1.3:
resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
engines: {node: ^16 || ^18 || >=20}
promise-limit@2.7.0:
resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==}
@ -10958,6 +10968,9 @@ packages:
resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==}
engines: {node: '>=14.16'}
tdigest@0.1.2:
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
telejson@7.2.0:
resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==}
@ -17993,6 +18006,8 @@ snapshots:
binary-extensions@2.2.0: {}
bintrees@1.0.2: {}
bl@4.1.0:
dependencies:
buffer: 5.7.1
@ -22811,6 +22826,11 @@ snapshots:
process@0.11.10: {}
prom-client@15.1.3:
dependencies:
'@opentelemetry/api': 1.9.0
tdigest: 0.1.2
promise-limit@2.7.0: {}
promise-polyfill@8.3.0: {}
@ -23952,6 +23972,10 @@ snapshots:
dependencies:
execa: 6.1.0
tdigest@0.1.2:
dependencies:
bintrees: 1.0.2
telejson@7.2.0:
dependencies:
memoizerific: 1.11.3