mirror of
https://activitypub.software/TransFem-org/Sharkey
synced 2024-09-29 11:25:39 +00:00
merge: Draft: Add /metrics endpoint (!613)
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/613
This commit is contained in:
commit
ea3c0e0bea
|
@ -3,6 +3,7 @@ stages:
|
|||
- deploy
|
||||
|
||||
testCommit:
|
||||
allow_failure: true
|
||||
stage: test
|
||||
image: node:iron
|
||||
services:
|
||||
|
@ -35,6 +36,31 @@ testCommit:
|
|||
- merge_requests
|
||||
- stable
|
||||
|
||||
Build Container for mr:
|
||||
stage: deploy
|
||||
parallel:
|
||||
matrix:
|
||||
- ARCH: amd64
|
||||
- ARCH: arm64
|
||||
tags:
|
||||
- ${ARCH}
|
||||
image:
|
||||
name: gcr.io/kaniko-project/executor:debug
|
||||
entrypoint: [""]
|
||||
before_script:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
|
||||
script:
|
||||
- >-
|
||||
/kaniko/executor
|
||||
--context "${CI_PROJECT_DIR}"
|
||||
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
||||
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
|
||||
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-latest
|
||||
--destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
|
||||
only:
|
||||
- merge_requests
|
||||
|
||||
getImageTag:
|
||||
stage: deploy
|
||||
image: ubuntu:latest
|
||||
|
|
|
@ -155,6 +155,7 @@
|
|||
"pg": "8.12.0",
|
||||
"pkce-challenge": "4.1.0",
|
||||
"probe-image-size": "7.2.3",
|
||||
"prom-client": "^15.1.3",
|
||||
"promise-limit": "2.7.0",
|
||||
"proxy-addr": "^2.0.7",
|
||||
"pug": "3.0.3",
|
||||
|
|
|
@ -12,6 +12,7 @@ import chalk from 'chalk';
|
|||
import chalkTemplate from 'chalk-template';
|
||||
import * as Sentry from '@sentry/node';
|
||||
import { nodeProfilingIntegration } from '@sentry/profiling-node';
|
||||
import { collectDefaultMetrics, AggregatorRegistry } from 'prom-client';
|
||||
import Logger from '@/logger.js';
|
||||
import { loadConfig } from '@/config.js';
|
||||
import type { Config } from '@/config.js';
|
||||
|
@ -94,6 +95,10 @@ export async function masterMain() {
|
|||
});
|
||||
}
|
||||
|
||||
// initialize prom-client in the master
|
||||
const aggregatorRegistry = new AggregatorRegistry();
|
||||
collectDefaultMetrics();
|
||||
|
||||
if (envOption.disableClustering) {
|
||||
if (envOption.onlyServer) {
|
||||
await server();
|
||||
|
|
|
@ -6,10 +6,10 @@
|
|||
import cluster from 'node:cluster';
|
||||
import * as Sentry from '@sentry/node';
|
||||
import { nodeProfilingIntegration } from '@sentry/profiling-node';
|
||||
import { collectDefaultMetrics, AggregatorRegistry } from 'prom-client';
|
||||
import { envOption } from '@/env.js';
|
||||
import { loadConfig } from '@/config.js';
|
||||
import { jobQueue, server } from './common.js';
|
||||
|
||||
/**
|
||||
* Init worker process
|
||||
*/
|
||||
|
@ -34,6 +34,10 @@ export async function workerMain() {
|
|||
});
|
||||
}
|
||||
|
||||
// initialize prom-client in the worker
|
||||
const aggregatorRegistry = new AggregatorRegistry();
|
||||
collectDefaultMetrics();
|
||||
|
||||
if (envOption.onlyServer) {
|
||||
await server();
|
||||
} else if (envOption.onlyQueue) {
|
||||
|
|
7
packages/backend/src/misc/metrics.ts
Normal file
7
packages/backend/src/misc/metrics.ts
Normal file
|
@ -0,0 +1,7 @@
|
|||
import { register, Counter, Gauge } from 'prom-client'
|
||||
|
||||
export const jobs_metrics = new Counter({
|
||||
name: "sharkey_jobs",
|
||||
help: "Metrics about jobs",
|
||||
labelNames: ['job', 'status', 'reason'] as const,
|
||||
});
|
|
@ -6,6 +6,7 @@
|
|||
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
|
||||
import * as Bull from 'bullmq';
|
||||
import * as Sentry from '@sentry/node';
|
||||
import { register, Gauge, Histogram } from 'prom-client';
|
||||
import type { Config } from '@/config.js';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type Logger from '@/logger.js';
|
||||
|
@ -142,6 +143,19 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
}
|
||||
}
|
||||
|
||||
const histogram = new Histogram({
|
||||
name: 'sharkey_queue_timings',
|
||||
help: 'Timings of jobs per queue',
|
||||
labelNames: ['queue', 'job'] as const,
|
||||
});
|
||||
|
||||
function runProcessorWithTimer(queue: String, processor: (job: Bull.Job) => void, job: Bull.Job) {
|
||||
const end = histogram.startTimer();
|
||||
processor(job);
|
||||
end({queue, job: job.name});
|
||||
}
|
||||
|
||||
|
||||
//#region system
|
||||
{
|
||||
const processer = (job: Bull.Job) => {
|
||||
|
@ -160,7 +174,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: System: ' + job.name }, () => processer(job));
|
||||
} else {
|
||||
return processer(job);
|
||||
return runProcessorWithTimer('system', processer, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.SYSTEM),
|
||||
|
@ -225,7 +239,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: DB: ' + job.name }, () => processer(job));
|
||||
} else {
|
||||
return processer(job);
|
||||
return runProcessorWithTimer('db', processer, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.DB),
|
||||
|
@ -257,7 +271,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: Deliver' }, () => this.deliverProcessorService.process(job));
|
||||
} else {
|
||||
return this.deliverProcessorService.process(job);
|
||||
return runProcessorWithTimer('deliver', this.deliverProcessorService.process, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.DELIVER),
|
||||
|
@ -297,7 +311,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: Inbox' }, () => this.inboxProcessorService.process(job));
|
||||
} else {
|
||||
return this.inboxProcessorService.process(job);
|
||||
return runProcessorWithTimer('inbox', this.inboxProcessorService.process, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.INBOX),
|
||||
|
@ -337,7 +351,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: UserWebhookDeliver' }, () => this.userWebhookDeliverProcessorService.process(job));
|
||||
} else {
|
||||
return this.userWebhookDeliverProcessorService.process(job);
|
||||
return runProcessorWithTimer('userwebhookdeliver', this.userWebhookDeliverProcessorService.process, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.USER_WEBHOOK_DELIVER),
|
||||
|
@ -377,7 +391,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: SystemWebhookDeliver' }, () => this.systemWebhookDeliverProcessorService.process(job));
|
||||
} else {
|
||||
return this.systemWebhookDeliverProcessorService.process(job);
|
||||
return runProcessorWithTimer('systemwebhookdeliver', this.systemWebhookDeliverProcessorService.process, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.SYSTEM_WEBHOOK_DELIVER),
|
||||
|
@ -427,7 +441,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: Relationship: ' + job.name }, () => processer(job));
|
||||
} else {
|
||||
return processer(job);
|
||||
return runProcessorWithTimer('relationship', processer, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.RELATIONSHIP),
|
||||
|
@ -472,7 +486,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: ObjectStorage: ' + job.name }, () => processer(job));
|
||||
} else {
|
||||
return processer(job);
|
||||
return runProcessorWithTimer('objectstorage', processer, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE),
|
||||
|
@ -505,7 +519,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
if (this.config.sentryForBackend) {
|
||||
return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job));
|
||||
} else {
|
||||
return this.endedPollNotificationProcessorService.process(job);
|
||||
return runProcessorWithTimer('endedpollnotification', this.endedPollNotificationProcessorService.process, job);
|
||||
}
|
||||
}, {
|
||||
...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),
|
||||
|
|
|
@ -24,6 +24,8 @@ import { bindThis } from '@/decorators.js';
|
|||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||
import type { DeliverJobData } from '../types.js';
|
||||
|
||||
import { jobs_metrics } from '@/misc/metrics.js'
|
||||
|
||||
@Injectable()
|
||||
export class DeliverProcessorService {
|
||||
private logger: Logger;
|
||||
|
@ -55,6 +57,7 @@ export class DeliverProcessorService {
|
|||
// ブロックしてたら中断
|
||||
const meta = await this.metaService.fetch();
|
||||
if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.toPuny(host))) {
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'skipped', reason: 'blocked' });
|
||||
return 'skip (blocked)';
|
||||
}
|
||||
|
||||
|
@ -69,6 +72,7 @@ export class DeliverProcessorService {
|
|||
this.suspendedHostsCache.set(suspendedHosts);
|
||||
}
|
||||
if (suspendedHosts.map(x => x.host).includes(this.utilityService.toPuny(host))) {
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'skipped', reason: 'suspended' });
|
||||
return 'skip (suspended)';
|
||||
}
|
||||
|
||||
|
@ -92,7 +96,7 @@ export class DeliverProcessorService {
|
|||
this.instanceChart.requestSent(i.host, true);
|
||||
}
|
||||
});
|
||||
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'success', reason: '' });
|
||||
return 'Success';
|
||||
} catch (res) {
|
||||
// Update stats
|
||||
|
@ -135,15 +139,19 @@ export class DeliverProcessorService {
|
|||
suspensionState: 'goneSuspended',
|
||||
});
|
||||
});
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: 'gone' });
|
||||
throw new Bull.UnrecoverableError(`${host} is gone`);
|
||||
}
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: res.statusCode });
|
||||
throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
|
||||
}
|
||||
|
||||
// 5xx etc.
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'failed', reason: 'remote_error' });
|
||||
throw new Error(`${res.statusCode} ${res.statusMessage}`);
|
||||
} else {
|
||||
// DNS error, socket error, timeout ...
|
||||
jobs_metrics.inc({ job: 'delivery', status: 'retrying', reason: 'remote_error' });
|
||||
throw res;
|
||||
}
|
||||
}
|
||||
|
|
33
packages/backend/src/server/MetricsServerService.ts
Normal file
33
packages/backend/src/server/MetricsServerService.ts
Normal file
|
@ -0,0 +1,33 @@
|
|||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import {
|
||||
AggregatorRegistry,
|
||||
collectDefaultMetrics,
|
||||
} from 'prom-client';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import type { FastifyInstance, FastifyPluginOptions } from 'fastify';
|
||||
|
||||
@Injectable()
|
||||
export class MetricsServerService {
|
||||
private register: AggregatorRegistry;
|
||||
|
||||
constructor(
|
||||
) {
|
||||
this.register = new AggregatorRegistry()
|
||||
collectDefaultMetrics({
|
||||
register: this.register,
|
||||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) {
|
||||
fastify.get('/', async (request, reply) => {
|
||||
reply
|
||||
.code(200)
|
||||
.type(this.register.contentType)
|
||||
.send(
|
||||
await this.register.clusterMetrics(),
|
||||
);
|
||||
});
|
||||
done();
|
||||
}
|
||||
}
|
|
@ -9,6 +9,7 @@ import { CoreModule } from '@/core/CoreModule.js';
|
|||
import { ApiCallService } from './api/ApiCallService.js';
|
||||
import { FileServerService } from './FileServerService.js';
|
||||
import { HealthServerService } from './HealthServerService.js';
|
||||
import { MetricsServerService } from './MetricsServerService.js';
|
||||
import { NodeinfoServerService } from './NodeinfoServerService.js';
|
||||
import { ServerService } from './ServerService.js';
|
||||
import { WellKnownServerService } from './WellKnownServerService.js';
|
||||
|
@ -60,6 +61,7 @@ import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js
|
|||
ClientLoggerService,
|
||||
FeedService,
|
||||
HealthServerService,
|
||||
MetricsServerService,
|
||||
UrlPreviewService,
|
||||
ActivityPubServerService,
|
||||
FileServerService,
|
||||
|
|
|
@ -29,6 +29,7 @@ import { StreamingApiServerService } from './api/StreamingApiServerService.js';
|
|||
import { WellKnownServerService } from './WellKnownServerService.js';
|
||||
import { FileServerService } from './FileServerService.js';
|
||||
import { HealthServerService } from './HealthServerService.js';
|
||||
import { MetricsServerService } from './MetricsServerService.js';
|
||||
import { ClientServerService } from './web/ClientServerService.js';
|
||||
import { OpenApiServerService } from './api/openapi/OpenApiServerService.js';
|
||||
import { MastodonApiServerService } from './api/mastodon/MastodonApiServerService.js';
|
||||
|
@ -65,6 +66,7 @@ export class ServerService implements OnApplicationShutdown {
|
|||
private nodeinfoServerService: NodeinfoServerService,
|
||||
private fileServerService: FileServerService,
|
||||
private healthServerService: HealthServerService,
|
||||
private metricsServerService: MetricsServerService,
|
||||
private clientServerService: ClientServerService,
|
||||
private globalEventService: GlobalEventService,
|
||||
private loggerService: LoggerService,
|
||||
|
@ -114,6 +116,8 @@ export class ServerService implements OnApplicationShutdown {
|
|||
fastify.register(this.oauth2ProviderService.createServer, { prefix: '/oauth' });
|
||||
fastify.register(this.healthServerService.createServer, { prefix: '/healthz' });
|
||||
|
||||
fastify.register(this.metricsServerService.createServer, { prefix: '/metrics' });
|
||||
|
||||
fastify.get<{ Params: { path: string }; Querystring: { static?: any; badge?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
|
||||
const path = request.params.path;
|
||||
|
||||
|
|
|
@ -355,6 +355,9 @@ importers:
|
|||
probe-image-size:
|
||||
specifier: 7.2.3
|
||||
version: 7.2.3
|
||||
prom-client:
|
||||
specifier: ^15.1.3
|
||||
version: 15.1.3
|
||||
promise-limit:
|
||||
specifier: 2.7.0
|
||||
version: 2.7.0
|
||||
|
@ -6006,6 +6009,9 @@ packages:
|
|||
resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
bintrees@1.0.2:
|
||||
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
|
||||
|
||||
bl@4.1.0:
|
||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||
|
||||
|
@ -9968,6 +9974,10 @@ packages:
|
|||
resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
|
||||
engines: {node: '>= 0.6.0'}
|
||||
|
||||
prom-client@15.1.3:
|
||||
resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
|
||||
engines: {node: ^16 || ^18 || >=20}
|
||||
|
||||
promise-limit@2.7.0:
|
||||
resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==}
|
||||
|
||||
|
@ -10958,6 +10968,9 @@ packages:
|
|||
resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
tdigest@0.1.2:
|
||||
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
|
||||
|
||||
telejson@7.2.0:
|
||||
resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==}
|
||||
|
||||
|
@ -17993,6 +18006,8 @@ snapshots:
|
|||
|
||||
binary-extensions@2.2.0: {}
|
||||
|
||||
bintrees@1.0.2: {}
|
||||
|
||||
bl@4.1.0:
|
||||
dependencies:
|
||||
buffer: 5.7.1
|
||||
|
@ -22811,6 +22826,11 @@ snapshots:
|
|||
|
||||
process@0.11.10: {}
|
||||
|
||||
prom-client@15.1.3:
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
tdigest: 0.1.2
|
||||
|
||||
promise-limit@2.7.0: {}
|
||||
|
||||
promise-polyfill@8.3.0: {}
|
||||
|
@ -23952,6 +23972,10 @@ snapshots:
|
|||
dependencies:
|
||||
execa: 6.1.0
|
||||
|
||||
tdigest@0.1.2:
|
||||
dependencies:
|
||||
bintrees: 1.0.2
|
||||
|
||||
telejson@7.2.0:
|
||||
dependencies:
|
||||
memoizerific: 1.11.3
|
||||
|
|
Loading…
Reference in a new issue