diff --git a/types/kafkajs/index.d.ts b/types/kafkajs/index.d.ts index 96a05a527a..13bb1dceed 100644 --- a/types/kafkajs/index.d.ts +++ b/types/kafkajs/index.d.ts @@ -7,528 +7,484 @@ /// import * as tls from "tls"; +import * as net from "net"; export class Kafka { - constructor(options: KafkaOptions); - - producer(options?: ProducerOptions): Producer; - consumer(options?: ConsumerOptions): Consumer; - admin(options?: AdminOptions): Admin; + constructor(config: KafkaConfig); + producer(config?: ProducerConfig): Producer; + consumer(config?: ConsumerConfig): Consumer; + admin(config?: AdminConfig): Admin; + logger(): Logger; } -export const PartitionAssigners: { - roundRobin: PartitionAssigner; -}; - -export namespace AssignerProtocol { - interface MemberMetadataOptions { - version: number; - topics: string[]; - userData?: Buffer; - } - - interface MemberMetadata { - encode(options: MemberMetadataOptions): Buffer; - decode(buffer: Buffer): MemberMetadataOptions; - } - - interface MemberAssignmentOptions { - version: number; - assignment: { [key: string]: number[] }; - userData?: Buffer; - } - - interface MemberAssignment { - encode(options: MemberAssignmentOptions): Buffer; - decode(buffer: Buffer): MemberAssignmentOptions; - } - - interface AssignerProtocolStatic { - MemberMetadata: MemberMetadata; - MemberAssignment: MemberAssignment; - } -} - -export const AssignerProtocol: AssignerProtocol.AssignerProtocolStatic; - -export enum CompressionTypes { - None = 0, - GZIP = 1, - Snappy = 2, - LZ4 = 3, - ZSTD = 4 -} - -export const CompressionCodecs: { [key in CompressionTypes]: () => any }; - -export enum ResourceTypes { - UNKNOWN = 0, - ANY = 1, - TOPIC = 2, - GROUP = 3, - CLUSTER = 4, - TRANSACTIONAL_ID = 5, - DELEGATION_TOKEN = 6 -} - -export interface LoggerMessage { - /** @var namespace Context from which the logger was called. */ - readonly namespace: string; - - /** @var level Logger level. */ - readonly level: logLevel; - - /** @var label Logger level label. */ - readonly label: string; - - /** @var log Content of the logger entry. */ - readonly log: LoggerMessageContent; -} - -export interface LoggerMessageContent { - /** @var timestamp Message timestamp. */ - readonly timestamp: Date; - - /** @var message Message sent to the logger. */ - readonly message: string; - - // Other possible fields in the content, that depend on the context. - [key: string]: any; -} - -export interface KafkaOptions { - clientId?: string; +export interface KafkaConfig { brokers: string[]; - ssl?: tls.ConnectionOptions; + ssl?: tls.SecureContextOptions; sasl?: SASLOptions; + clientId?: string; connectionTimeout?: number; + authenticationTimeout?: number; requestTimeout?: number; + enforceRequestTimeout?: boolean; retry?: RetryOptions; + socketFactory?: ISocketFactory; logLevel?: logLevel; - logCreator?: () => (message: LoggerMessage) => void; + logCreator?: logCreator; } +export type ISocketFactory = (host: string, port: number, ssl: tls.SecureContextOptions, onConnect: () => void) => net.Socket; + export interface SASLOptions { mechanism: "plain" | "scram-sha-256" | "scram-sha-512"; username: string; password: string; } +export interface ProducerConfig { + createPartitioner?: ICustomPartitioner; + retry?: RetryOptions; + metadataMaxAge?: number; + allowAutoTopicCreation?: boolean; + idempotent?: boolean; + transactionalId?: string; + transactionTimeout?: number; + maxInFlightRequests?: number; +} + +export type ICustomPartitioner = () => ( + message: { topic: string; partitionMetadata: PartitionMetadata[]; message: Message }, +) => number; + +export interface Message { + key?: string | Buffer; + value: string | Buffer | null; + partition?: string | number; + headers?: IHeaders; + timestamp?: number | string; +} + +export interface PartitionMetadata { + partitionErrorCode: number; + partitionId: number; + leader: number; + replicas: number[]; + isr: number[]; +} + +// tslint:disable-next-line:interface-name +export interface IHeaders { + [key: string]: string; +} + +export interface ConsumerConfig { + groupId: string; + partitionAssigners?: PartitionAssigner[]; + metadataMaxAge?: number; + sessionTimeout?: number; + rebalanceTimeout?: number; + heartbeatInterval?: number; + maxBytesPerPartition?: number; + minBytes?: number; + maxBytes?: number; + maxWaitTimeInMs?: number; + retry?: RetryOptions; + allowAutoTopicCreation?: boolean; + maxInFlightRequests?: number; + readUncommitted?: boolean; +} + +export interface PartitionAssigner { + new (config: { cluster: Cluster }): Assigner; +} + +export interface Cluster { + isConnected(): void; + connect(): Promise; + disconnect(): Promise; + refreshMetadata(): Promise; + refreshMetadataIfNecessary(): Promise; + addTargetTopic(topic: string): Promise; + findBroker(node: { nodeId: string }): Promise; + findControllerBroker(): Promise; + findTopicPartitionMetadata(topic: string): PartitionMetadata[]; + findLeaderForPartitions(topic: string, partitions: number[]): { [leader: string]: number[] }; + findGroupCoordinator(group: { groupId: string }): Promise; + findGroupCoordinatorMetadata(group: { + groupId: string; + }): Promise<{ errorCode: number; coordinator: { nodeId: number; host: string; port: number } }>; + defaultOffset(config: { fromBeginning: boolean }): number; + fetchTopicsOffset( + topics: Array<{ topic: string; partitions: Array<{ partition: number }>; fromBeginning: boolean }>, + ): Promise<{ topic: string; partitions: Array<{ partition: number; offset: string }> }>; +} + +export interface Assignment { [topic: string]: number[]; } + +export interface GroupMember { memberId: string; } + +export interface GroupMemberAssignment { memberId: string; memberAssignment: Buffer; } + +export interface GroupState { name: string; metadata: Buffer; } + +export interface Assigner { + name: string; + version: number; + assign(group: { + members: GroupMember[]; + topics: string[], + userData: Buffer, + }): Promise; + protocol(subscription: { topics: string[], userData: Buffer }): GroupState; +} + export interface RetryOptions { maxRetryTime?: number; initialRetryTime?: number; factor?: number; multiplier?: number; retries?: number; - maxInFlightRequests?: number | null; } +export interface AdminConfig { + retry?: RetryOptions; +} + +// tslint:disable-next-line:interface-name +export interface ITopicConfig { + topic: string; + numPartitions?: number; + replicationFactor?: number; + replicaAssignment?: object[]; + configEntries?: object[]; +} + +// tslint:disable-next-line:interface-name +export interface ITopicMetadata { + topic: string; + partitions: PartitionMetadata[]; +} + +export enum ResourceType { + UNKNOWN = 0, + ANY = 1, + TOPIC = 2, + GROUP = 3, + CLUSTER = 4, + TRANSACTIONAL_ID = 5, + DELEGATION_TOKEN = 6, +} + +export interface ResourceConfigQuery { + type: ResourceType; + name: string; + configNames: string[]; +} + +export interface ConfigEntries { + configName: string; + configValue: string; + isDefault: boolean; + isSensitive: boolean; + readOnly: boolean; + configSynonyms: ConfigSynonyms[]; +} + +export interface ConfigSynonyms { + configName: string; + configValue: string; + configSource: number; +} + +export interface DescribeConfigResponse { + resources: Array<{ + configEntries: ConfigEntries[], + errorCode: number, + errorMessage: string, + resourceName: string, + resourceType: ResourceType, + }>; + throttleTime: number; +} + +// tslint:disable-next-line:interface-name +export interface IResourceConfig { + type: ResourceType; + name: string; + configEntries: Array<{ name: string, value: string }>; +} + +export type ValueOf = T[keyof T]; + +export interface AdminEvents { + CONNECT: "admin.connect"; + DISCONNECT: "admin.disconnect"; + REQUEST: "admin.network.request"; + REQUEST_TIMEOUT: "admin.network.request_timeout"; + REQUEST_QUEUE_SIZE: "admin.network.request_queue_size"; +} + +export interface InstrumentationEvent { + id: string; + type: string; + timestamp: number; + payload: T; +} + +export type ConnectEvent = InstrumentationEvent; +export type DisconnectEvent = InstrumentationEvent; +export type RequestEvent = InstrumentationEvent<{ + apiKey: number, + apiName: string, + apiVersion: number, + broker: string, + clientId: string, + correlationId: number, + createdAt: number, + duration: number, + pendingDuration: number, + sentAt: number, + size: number, +}>; +export type RequestTimeoutEvent = InstrumentationEvent<{ + apiKey: number, + apiName: string, + apiVersion: number, + broker: string, + clientId: string, + correlationId: number, + createdAt: number, + pendingDuration: number, + sentAt: number, +}>; +export type RequestQueueSizeEvent = InstrumentationEvent<{ + broker: string, + clientId: string, + queueSize: number, +}>; + +export interface SeekEntry { + partition: number; + offset: string; +} + +export interface Admin { + connect(): Promise; + disconnect(): Promise; + createTopics(options: { + validateOnly?: boolean; + waitForLeaders?: boolean; + timeout?: number; + topics: ITopicConfig[]; + }): Promise; + deleteTopics(topics: { topics: string[], timeout: number}): Promise; + fetchTopicMetadata(topicMetadata: ITopicMetadata): Promise; + fetchOffsets(topic: { groupId: string; topic: string }): Promise>; + fetchTopicOffsets(topic: string): Promise; + setOffsets(topic: { groupId: string; topic: string; partitions: SeekEntry[] }): Promise; + resetOffsets(topic: { groupId: string; topic: string; earliest: boolean }): Promise; + describeConfigs(configs: { + resources: ResourceConfigQuery[], + includeSynonyms: boolean, + }): Promise; + alterConfigs(configs: { + validateOnly: boolean, + resources: IResourceConfig[], + }): Promise; + logger(): Logger; + on(eventName: ValueOf, listener: (...args: any[]) => void): void; + events: AdminEvents; +} + +export const PartitionAssigners: { roundRobin: PartitionAssigner }; + +// tslint:disable-next-line:interface-name +export interface ISerializer { + encode(value: T): Buffer; + decode(buffer: Buffer): T; +} + +export interface MemberMetadata { + version: number; + topics: string[]; + userData: Buffer; +} + +export interface MemberAssignment { + version: number; + assignment: Assignment; + userData: Buffer; +} + +export const AssignerProtocol: { + MemberMetadata: ISerializer; + MemberAssignment: ISerializer; +}; + +export type DefaultPartitioner = ( + message: { topic: string; partitionMetadata: PartitionMetadata[]; message: Message }, +) => number; + +export type JavaCompatiblePartitioner = ( + message: { topic: string; partitionMetadata: PartitionMetadata[]; message: Message }, +) => number; + +export const Partitioners: { + DefaultPartitioner: DefaultPartitioner, + JavaCompatiblePartitioner: JavaCompatiblePartitioner, +}; + export enum logLevel { NOTHING = 0, ERROR = 1, WARN = 2, INFO = 4, - DEBUG = 5 + DEBUG = 5, } -export interface Producer { +export interface LogEntry { namespace: string; level: logLevel; label: string; log: string; } + +export type Logger = (entry: LogEntry) => void; + +export type logCreator = (logLevel: string) => ( + namespace: string, level: string, label: string, log: string, +) => void; + +export interface Broker { + isConnected(): boolean; connect(): Promise; disconnect(): Promise; - - send(payload: MessagePayload): Promise; - sendBatch(payload: MessageBatchPayload): Promise; - - transaction(): Promise; - - events: ProducerEvents; - on( - event: ProducerEvents[keyof ProducerEvents], - cb: (e: InstrumentationEvent) => void - ): () => Producer; + apiVersions(): Promise<{ [apiKey: number]: { minVersion: number; maxVersion: number } }>; + metadata( + topics: string[], + ): Promise<{ + brokers: Array<{ nodeId: number; host: string; port: number }>; + topicMetadata: Array<{ topicErrorCode: number; topic: number; partitionMetadata: PartitionMetadata[] }>; + }>; + offsetCommit(request: { + groupId: string; + groupGenerationId: number; + memberId: string; + retentionTime?: number; + topics: Array<{ topic: string; partitions: Array<{ partition: number; offset: string }> }>; + }): Promise; } -export interface ProducerOptions { - createPartitioner?: () => (options: { - topic: string; - partitionMetadata: PartitionMetadata[]; - message: ProducerMessage; - }) => number; - retry?: RetryOptions; - metadataMaxAge?: number; - allowAutoTopicCreation?: boolean; - transactionTimeout?: number; - idempotent?: boolean; +export interface KafkaMessage { + key: Buffer; + value: Buffer; + timestamp: string; + size: number; + attributes: number; + offset: string; + headers?: IHeaders; } -export interface PartitionerPartitionMetadata { - partitionId: number; - leader: number; -} - -export interface PartitionMetadata { - partitionId: number; - leader: number; - partitionErrorCode?: number; - replicas?: number[]; - isr?: number[]; -} - -export interface MessagePayloadBase { - acks?: AcksBehaviour; +export interface ProducerRecord { + topic: string; + messages: Message[]; + acks?: number; timeout?: number; compression?: CompressionTypes; } -export interface MessagePayload extends MessagePayloadBase { - topic: string; - messages: ProducerMessage[]; - transactionTimeout?: number; - idempotent?: boolean; -} - -export interface MessageBatchPayload extends MessagePayloadBase { - topicMessages: ProducerTopicMessage[]; -} - -export interface ProducerMessage { - partition?: number; - key?: string; - value: string | Buffer | ArrayBuffer; - headers?: { [key: string]: string }; -} - -export interface ProducerTopicMessage { - topic: string; - messages: ProducerMessage[]; -} - -export enum AcksBehaviour { - All = -1, - No = 0, - Leader = 1 -} - -export interface Transaction { - send(payload: MessagePayload): Promise; - sendBatch(payload: MessageBatchPayload): Promise; - - sendOffsets(offsets: TransactionSendOffsets): Promise; - - commit(): Promise; - abort(): Promise; -} - -export interface TransactionSendOffsets { - consumerGroupId: string; - topics: TransactionSendOffsetsTopic[]; -} - -export interface TransactionSendOffsetsTopic { - topic: string; - partitions: TransactionSendOffsetsTopicPartitions[]; -} - -export interface TransactionSendOffsetsTopicPartitions { +export interface RecordMetadata { + topicName: string; partition: number; + errorCode: number; offset: string; + timestamp: string; } -export interface Consumer { - connect(): Promise; - disconnect(): Promise; - - subscribe(options: ConsumerSubscribeOptions): Promise; - - run(options: ConsumerRunOptions): Promise; - - pause(topics: Array<{ topic: string }>): void; - resume(topics: Array<{ topic: string }>): void; - seek(options: ConsumerSeekOptions): void; - - describeGroup(): Promise; - - events: ConsumerEvents; - on( - event: ConsumerEvents[keyof ConsumerEvents], - cb: (e: InstrumentationEvent) => void - ): () => Consumer; -} - -export interface ConsumerOptions { - groupId: string; - partitionAssigners?: PartitionAssigner[]; - sessionTimeout?: number; - heartbeatInterval?: number; - metadataMaxAge?: number; - allowAutoTopicCreation?: boolean; - maxBytesPerPartition?: number; - minBytes?: number; - maxBytes?: number; - maxWaitTimeInMs?: number; - retry?: RetryOptions; - readUncommitted?: boolean; -} - -export interface PartitionAssigner { - ({ cluster }: { cluster: any /* TODO */ }): { - name: string; - version: number; - assign: (options: { - members: Array<{ memberId: string }>; - topics: any[]; - userData?: Buffer; - }) => Promise< - Array<{ - memberId: number; - memberAssignment: Buffer; - }> - >; - protocol?: (options: { - topics: any /* TODO */; - }) => { name: string; metadata: Buffer }; - }; -} - -export interface ConsumerRunOptions { - eachMessage?: (payload: ConsumerEachMessagePayload) => Promise; - eachBatch?: (payload: ConsumerEachBatchPayload) => Promise; - eachBatchAutoResolve?: boolean; - autoCommitInterval?: number; - autoCommitThreshold?: number; - autoCommit?: boolean; -} - -export interface ConsumerSubscribeOptions { +export interface TopicMessages { topic: string; - fromBeginning?: boolean; + messages: Message[]; } -export interface ConsumerMessage { - timestamp: number; - key: string; - value: Buffer; - headers: { [key: string]: string }; - offset: number; -} - -export interface ConsumerBatch { - topic: string; - partition: number; - highWatermark: number; - messages: ConsumerMessage[]; -} - -export interface ConsumerEachMessagePayload { - topic: string; - partition: number; - message: ConsumerMessage; -} - -export interface ConsumerEachBatchPayload { - batch: ConsumerBatch; - resolveOffset: (offset: number) => Promise; - heartbeat: () => Promise; - isRunning: () => boolean; - commitOffsetsIfNecessary: ( - offsets?: OffsetsByTopicPartition - ) => Promise; - uncommittedOffsets: () => OffsetsByTopicPartition; -} - -export interface OffsetsByTopicPartition { - topics: TopicOffsets[]; -} - -export interface TopicOffsets { - partitions: PartitionOffset[]; +export interface ProducerBatch { + acks: number; + timeout: number; + compression: CompressionTypes; + topicMessages: TopicMessages[]; } export interface PartitionOffset { - partition: string; + partition: number; offset: string; } -export interface ConsumerSeekOptions { +export interface TopicOffsets { topic: string; - partition: number; - offset: number; + partitions: PartitionOffset[]; } -export interface GroupMemberMetadata { - memberId: string; - clientId: string; - clientHost: string; - memberMetadata: Buffer; - memberAssignment: Buffer; +export interface Offsets { + topics: TopicOffsets[]; } -export interface GroupMetadata { - errorCode: number; +export interface Sender { + send(record: ProducerRecord): Promise; + sendBatch(batch: ProducerBatch): Promise; +} + +export interface ProducerEvents { + CONNECT: "producer.connect"; + DISCONNECT: "producer.disconnect"; + REQUEST: "producer.network.request"; + REQUEST_TIMEOUT: "producer.network.request_timeout"; + REQUEST_QUEUE_SIZE: "producer.network.request_queue_size"; +} + +export type Producer = Sender & { + connect(): Promise; + disconnect(): Promise; + isIdempotent(): boolean; + events: ProducerEvents; + on(eventName: ValueOf, listener: (...args: any[]) => void): void; + transaction(): Promise; + logger(): Logger; +}; + +export type Transaction = Sender & { + sendOffsets(offsets: Offsets & { consumerGroupId: string }): Promise; + commit(): Promise; + abort(): Promise; + isActive(): boolean; +}; + +export interface ConsumerGroup { groupId: string; - protocolType: string; + generationId: number; + memberId: string; + coordinator: Broker; +} + +export interface MemberDescription { + clientHost: string; + clientId: string; + memberId: string; + memberAssignment: Buffer; + memberMetadata: Buffer; +} + +export interface GroupDescription { + groupId: string; + members: MemberDescription[]; protocol: string; - members: GroupMemberMetadata[]; + protocolType: string; state: string; } -export interface Admin { - connect(): Promise; - disconnect(): Promise; +export interface TopicPartitions { topic: string; partitions: number[]; } - createTopics(options: AdminCreateTopicsOptions): Promise; - deleteTopics(options: AdminDeleteTopicsOptions): Promise; - getTopicMetadata(options: { - topics?: string[]; - }): Promise<{ topics: TopicMetadata[] }>; - - fetchOffsets( - options: AdminFetchOffsetsOptions - ): Promise; - resetOffsets(options: AdminResetOffsetsOptions): Promise; - setOffsets(options: AdminSetOffsetsOptions): Promise; - - describeConfigs( - options: AdminDescribeConfigsOptions - ): Promise; - alterConfigs(options: AdminAlterConfigsOptions): Promise; - - events: AdminEvents; - on( - event: AdminEvents[keyof AdminEvents], - cb: (e: InstrumentationEvent) => void - ): () => Admin; -} - -export interface AdminOptions { - retry?: RetryOptions; -} - -export interface AdminCreateTopicsOptions { - validateOnly?: boolean; - waitForLeaders?: boolean; - timeout?: number; - topics: AdminTopic[]; -} - -export interface AdminTopic { +export interface Batch { topic: string; - numPartitions?: number; - replicationFactor?: number; - replicaAssignment?: AdminTopicReplicaAssignment[]; - configEntries?: AdminTopicConfigEntry[]; -} - -export interface AdminTopicReplicaAssignment { partition: number; - replicas: number[]; -} - -export interface AdminTopicConfigEntry { - name: string; - value: string; -} - -export interface AdminDeleteTopicsOptions { - timeout?: number; - topics: string[]; -} - -export interface AdminFetchOffsetsOptions { - groupId: string; - topic: string; -} - -export interface AdminResetOffsetsOptions { - groupId: string; - topic: string; - earliest?: boolean; -} - -export interface TopicMetadata { - name: string; - partitions: PartitionMetadata[]; -} - -export interface AdminDescribeConfigsOptions { - resources: ResourceConfigQuery[]; -} - -export interface ResourceConfigQuery { - type: ResourceTypes; - name: string; - configNames?: string[]; -} - -export interface AdminConfigDescription { - resources: AdminConfigDescriptionResource[]; - throttleTime: number; -} - -export interface AdminConfigDescriptionResource { - configEntries: AdminConfigDescriptionResourceConfigEntry[]; - errorCode: number; - errorMessage: string; - resourceName: string; - resourceType: ResourceTypes; -} - -export interface AdminConfigDescriptionResourceConfigEntry { - configName: string; - configValue: string; - isDefault: boolean; - isSensitive: boolean; - readOnly: boolean; -} - -export interface AdminAlterConfigsOptions { - validateOnly: boolean; - resources: ResourceConfigQuery[]; -} - -export interface ResourceConfigQuery { - type: ResourceTypes; - name: string; - configEntries: ResourceConfigEntry[]; -} - -export interface ResourceConfigEntry { - name: string; - value: string; -} - -export interface AdminAlterConfigReturn { - resources: AdminAlterConfigResource[]; - throttleTime: number; -} - -export interface AdminAlterConfigResource { - errorCode: number; - errorMessage: string; - resourceName: string; - resourceType: ResourceTypes; -} - -export interface AdminTopicOffset { - partition: number; - offset: string; -} - -export interface AdminSetOffsetsSeekEntry { - partition: number; - offset: string; -} - -export interface AdminSetOffsetsOptions { - groupId: string; - topic: string; - partitions: AdminSetOffsetsSeekEntry[]; -} - -export interface InstrumentationEvent { - id: number; - type: string; - timestamp: number; - payload: { [key: string]: any }; + highWatermark: string; + messages: KafkaMessage[]; + isEmpty(): boolean; + firstOffset(): string | null; + lastOffset(): string; + offsetLag(): string; } export interface ConsumerEvents { @@ -537,28 +493,110 @@ export interface ConsumerEvents { GROUP_JOIN: "consumer.group_join"; FETCH: "consumer.fetch"; START_BATCH_PROCESS: "consumer.start_batch_process"; - END_BATCH_PROCESS: "consumner.end_batch_process"; + END_BATCH_PROCESS: "consumer.end_batch_process"; CONNECT: "consumer.connect"; DISCONNECT: "consumer.disconnect"; STOP: "consumer.stop"; CRASH: "consumer.crash"; - REQUEST: "consumer.request"; - REQUEST_TIMEOUT: "consumer.request_timeout"; - REQUEST_QUEUE_SIZE: "consumer.request_queue_size"; + REQUEST: "consumer.network.request"; + REQUEST_TIMEOUT: "consumer.network.request_timeout"; + REQUEST_QUEUE_SIZE: "consumer.network.request_queue_size"; +} +export type ConsumerHeartbeatEvent = InstrumentationEvent<{ + groupId: string, + memberId: string, + groupGenerationId: number, +}>; +export type ConsumerCommitOffsetsEvent = InstrumentationEvent<{ + groupId: string, + memberId: string, + groupGenerationId: number, + topics: Array<{ + topic: string, + partitions: Array<{ + offset: string, + partition: string, + }> + }>, +}>; +// tslint:disable-next-line:interface-name +export interface IMemberAssignment { + [key: string]: number[]; +} +export type ConsumerGroupJoinEvent = InstrumentationEvent<{ + duration: number, + groupId: string, + isLeader: boolean, + leaderId: string, + groupProtocol: string, + memberId: string, + memberAssignment: IMemberAssignment; +}>; +export type ConsumerFetchEvent = InstrumentationEvent<{ + numberOfBatches: number, + duration: number, +}>; + +// tslint:disable-next-line:interface-name +export interface IBatchProcessEvent { + topic: string; + partition: number; + highWatermark: string; + offsetLag: string; + batchSize: number; + firstOffset: string; + lastOffset: string; +} +export type ConsumerStartBatchProcessEvent = InstrumentationEvent; +export type ConsumerEndBatchProcessEvent = InstrumentationEvent; +export type ConsumerCrashEvent = InstrumentationEvent<{ + error: Error, + groupId: string, +}>; + +export interface Consumer { + connect(): Promise; + disconnect(): Promise; + subscribe(topic: { topic: string; fromBeginning?: boolean }): Promise; + stop(): Promise; + run(config?: { + autoCommit?: boolean; + autoCommitInterval?: number | null; + autoCommitThreshold?: number | null; + eachBatchAutoResolve?: boolean; + partitionsConsumedConcurrently?: number; + eachBatch?: ( + batch: { + batch: Batch; + resolveOffset(offset: string): void; + heartbeat(): Promise; + commitOffsetsIfNecessary(offsets?: Offsets): Promise; + uncommittedOffsets(): Promise; + isRunning(): boolean; + }, + ) => Promise; + eachMessage?: (message: { topic: string; partition: number; message: KafkaMessage }) => Promise; + }): Promise; + seek(topicPartition: { topic: string; partition: number; offset: string }): void; + describeGroup(): Promise; + pause(topicPartitions: TopicPartitions[]): void; + resume(topicPartitions: TopicPartitions[]): void; + on(eventName: ValueOf, listener: (...args: any[]) => void): void; + logger(): Logger; + events: ConsumerEvents; } -export interface ProducerEvents { - CONNECT: "producer.connect"; - DISCONNECT: "producer.disconnect"; - REQUEST: "producer.request"; - REQUEST_TIMEOUT: "producer.request_timeout"; - REQUEST_QUEUE_SIZE: "producer.request_queue_size"; +export enum CompressionTypes { + None = 0, + GZIP = 1, + Snappy = 2, + LZ4 = 3, + ZSTD = 4, } -export interface AdminEvents { - CONNECT: "admin.connect"; - DISCONNECT: "admin.disconnect"; - REQUEST: "admin.request"; - REQUEST_TIMEOUT: "admin.request_timeout"; - REQUEST_QUEUE_SIZE: "admin.request_queue_size"; -} +export const CompressionCodecs: { + [CompressionTypes.GZIP]: () => any; + [CompressionTypes.Snappy]: () => any; + [CompressionTypes.LZ4]: () => any; + [CompressionTypes.ZSTD]: () => any; +}; diff --git a/types/kafkajs/kafkajs-tests.ts b/types/kafkajs/kafkajs-tests.ts index 15fe47deba..092cc887d8 100644 --- a/types/kafkajs/kafkajs-tests.ts +++ b/types/kafkajs/kafkajs-tests.ts @@ -2,42 +2,30 @@ import * as fs from "fs"; import { Kafka, - AssignerProtocol, PartitionAssigners, logLevel, CompressionTypes, - CompressionCodecs, - ResourceTypes, - PartitionAssigner, - LoggerMessage + CompressionCodecs } from "kafkajs"; -const { MemberMetadata, MemberAssignment } = AssignerProtocol; const { roundRobin } = PartitionAssigners; // COMMON const host = "localhost"; const topic = "topic-test"; -const logger = (loggerMessage: LoggerMessage): void => { - console.log(`[${loggerMessage.namespace}] ${loggerMessage.log.message}`); -}; - const kafka = new Kafka({ logLevel: logLevel.INFO, brokers: [`${host}:9094`, `${host}:9097`, `${host}:9100`], clientId: "example-consumer", ssl: { - servername: "localhost", - rejectUnauthorized: false, ca: [fs.readFileSync("./testHelpers/certs/cert-signed", "utf-8")] }, sasl: { mechanism: "plain", username: "test", password: "testtest" - }, - logCreator: () => logger + } }); // CONSUMER @@ -47,13 +35,20 @@ const runConsumer = async () => { await consumer.connect(); await consumer.subscribe({ topic }); await consumer.run({ - // eachBatch: async ({ batch }) => { - // console.log(batch) - // }, + eachBatch: async ({ commitOffsetsIfNecessary }) => { + commitOffsetsIfNecessary({ + topics: [{ + topic: 'topic-name', + partitions: [ + { partition: 0, offset: '500' } + ] + }] + }); + }, eachMessage: async ({ topic, partition, message }) => { const prefix = `${topic}[${partition} | ${message.offset}] / ${ message.timestamp - }`; + }`; console.log(`- ${prefix} ${message.key}#${message.value}`); } }); @@ -96,10 +91,17 @@ runProducer().catch(e => console.error(`[example/producer] ${e.message}`, e)); const admin = kafka.admin({ retry: { retries: 10 } }); const runAdmin = async () => { - await admin.connect(); - const { topics } = await admin.getTopicMetadata({}); - await admin.createTopics({ topics: [{ topic, numPartitions: 10, replicationFactor: 1}], timeout: 30000, waitForLeaders: true }); - await admin.disconnect(); + await admin.connect(); + await admin.fetchTopicMetadata({ + topic: 'string', + partitions: [] + }); + await admin.createTopics({ + topics: [{ topic, numPartitions: 10, replicationFactor: 1 }], + timeout: 30000, + waitForLeaders: true + }); + await admin.disconnect(); }; runAdmin().catch(e => console.error(`[example/admin] ${e.message}`, e)); @@ -113,39 +115,10 @@ async () => { }); }; -// import SnappyCodec from "kafkajs-snappy"; const SnappyCodec: any = undefined; CompressionCodecs[CompressionTypes.Snappy] = SnappyCodec; -const myCustomAssignmentArray = [0]; -const assignment: { [key: number]: { [key: string]: number[] } } = { - 0: { a: [0] } -}; -const MyPartitionAssigner: PartitionAssigner = ({ cluster: any }) => ({ - name: "MyPartitionAssigner", - version: 1, - async assign({ members, topics }) { - // perform assignment - return myCustomAssignmentArray.map(memberId => ({ - memberId, - memberAssignment: MemberAssignment.encode({ - version: this.version, - assignment: assignment[memberId] - }) - })); - }, - protocol({ topics }) { - return { - name: this.name, - metadata: MemberMetadata.encode({ - version: this.version, - topics - }) - }; - } -}); - kafka.consumer({ groupId: "my-group", - partitionAssigners: [MyPartitionAssigner, roundRobin] + partitionAssigners: [roundRobin] });