Check messages aganist OpenAI moderation API

src/index.ts

@@ -3,6 +3,7 @@ import { Configuration as OpenAIApiConfiguration, OpenAIApi } from "openai";
 
 import config from "./config.json";
 import toOpenAIMessages from "./toOpenAIMessages";
+import Moderation from "./moderation";
 
 const discord = new DiscordApi.Client({
   intents: [
@@ -12,7 +13,7 @@ const discord = new DiscordApi.Client({
   ]
 });
 
-const openai = new OpenAIApi(new OpenAIApiConfiguration({
+export const openai = new OpenAIApi(new OpenAIApiConfiguration({
   apiKey: config.tokens.OpenAI
 }));
 
@@ -29,6 +30,8 @@ discord.on("messageCreate", async message => {
 
     messages = messages.filter(m => message.createdTimestamp - m.createdTimestamp < 1000*60*60 );
 
+    messages.forEach(m => Moderation.checkMessage(m));
+
     message.channel.sendTyping();
     const answer = await openai.createChatCompletion({
       model: "gpt-3.5-turbo",
@@ -36,7 +39,7 @@ discord.on("messageCreate", async message => {
       max_tokens: 168
     });
 
-    message.reply({
+    const response = message.reply({
       content: answer.data.choices[0].message?.content,
       allowedMentions: {
         repliedUser: false,
@@ -48,6 +51,8 @@ discord.on("messageCreate", async message => {
       const channelName: string = message.inGuild() ? `${message.channel.name} (${message.guild.name})` : `@${message.author.tag}`;
       console.log(`Used ${usage.total_tokens} (${usage.prompt_tokens} + ${usage.completion_tokens}) tokens for ${message.author.tag} (${message.author.id}) in #${channelName}`);
     }
+
+    Moderation.checkMessage(await response);
   } catch (e) {
     console.error(e);
 

src/moderation.ts

@@ -0,0 +1,31 @@
+import { Collection, Message } from "discord.js";
+import { openai } from "./index";
+import { formatMessage } from "./toOpenAIMessages";
+
+export default class Moderation {
+  /** Represents cache of messages that have been checked aganist OpenAI moderation API. */
+  private static cache = new Collection<string, boolean>();
+
+  public static async checkMessage(message: Message): Promise<boolean> {
+    if (this.cache.has(message.id)) {
+      return this.cache.get(message.id) as boolean;
+    }
+
+    const warningReaction = message.reactions.resolve("⚠");
+    // if bot reacted to that message, we already know that it returned true for moderation API
+    if (warningReaction && warningReaction.me) {
+      this.cache.set(message.id, true);
+      return true;
+    }
+
+    const answer = await openai.createModeration({
+      input: formatMessage(message),
+    });
+
+    const flagged = answer.data.results[0].flagged;
+    this.cache.set(message.id, flagged);
+    if (flagged) message.react("⚠");
+
+    return flagged;
+  }
+}

src/toOpenAIMessages.ts

@@ -3,7 +3,7 @@ import { Collection, Message as DiscordMessage } from "discord.js";
 
 import config from "./config.json";
 
-function formatMessage(message: DiscordMessage): string {
+export function formatMessage(message: DiscordMessage): string {
   let rvalue: string = message.cleanContent;
 
   for (const attachment of message.attachments) {