Content Moderation

import OpenAI from "openai";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
});

// Categories returned by the omni-moderation model. Derived from the SDK so the
// shape (including `boolean | null` for the illicit categories) stays accurate.
export type ModerationCategories = OpenAI.Moderation["categories"];
export type ModerationCategoryScores = OpenAI.Moderation["category_scores"];

export interface ModerationResult {
  flagged: boolean;
  categories: ModerationCategories;
  categoryScores: ModerationCategoryScores;
}

// Moderate text content using OpenAI
export async function moderateText(text: string): Promise<ModerationResult> {
  const response = await openai.moderations.create({
    model: "omni-moderation-latest",
    input: text,
  });
  const result = response.results[0];

  return {
    flagged: result.flagged,
    categories: result.categories,
    categoryScores: result.category_scores,
  };
}

// Check if content is safe
export async function isSafe(text: string): Promise<boolean> {
  const result = await moderateText(text);
  return !result.flagged;
}

// Get detailed violation info
export async function getViolations(text: string): Promise<{ category: string; score: number }[]> {
  const result = await moderateText(text);
  if (!result.flagged) return [];

  return Object.entries(result.categories)
    .filter(([, flagged]) => flagged)
    .map(([category]) => ({
      category,
      score: result.categoryScores[category as keyof ModerationCategoryScores],
    }))
    .sort((a, b) => b.score - a.score);
}

import type { ModerationCategories, ModerationCategoryScores } from "./text";

export interface ModerationPolicy {
  name: string;
  blockedCategories: Partial<Record<keyof ModerationCategories, boolean>>;
  thresholds?: Partial<Record<keyof ModerationCategories, number>>;
  action: "block" | "review" | "warn";
}

export const policies: Record<string, ModerationPolicy> = {
  // Strict policy for public content
  strict: {
    name: "Strict",
    blockedCategories: {
      sexual: true,
      hate: true,
      harassment: true,
      "self-harm": true,
      violence: true,
    },
    action: "block",
  },

  // Moderate policy for community content
  moderate: {
    name: "Moderate",
    blockedCategories: {
      "sexual/minors": true,
      "hate/threatening": true,
      "violence/graphic": true,
    },
    thresholds: {
      sexual: 0.8,
      hate: 0.7,
      harassment: 0.7,
    },
    action: "review",
  },

  // Permissive policy for mature platforms
  permissive: {
    name: "Permissive",
    blockedCategories: {
      "sexual/minors": true,
    },
    action: "warn",
  },
};

export function checkPolicy(
  policy: ModerationPolicy,
  categories: ModerationCategories,
  scores: ModerationCategoryScores
): { passed: boolean; violations: string[]; action: ModerationPolicy["action"] | "allow" } {
  const violations: string[] = [];

  for (const [category, blocked] of Object.entries(policy.blockedCategories)) {
    if (blocked && categories[category as keyof ModerationCategories]) {
      violations.push(category);
    }
  }

  if (policy.thresholds) {
    for (const [category, threshold] of Object.entries(policy.thresholds)) {
      if (scores[category as keyof ModerationCategoryScores] >= threshold) {
        if (!violations.includes(category)) violations.push(category);
      }
    }
  }

  // Clean content is allowed; only apply the policy's action on a violation.
  return {
    passed: violations.length === 0,
    violations,
    action: violations.length > 0 ? policy.action : "allow",
  };
}

import { NextRequest, NextResponse } from "next/server";
import { moderateText } from "@/lib/moderation/text";
import { policies, checkPolicy } from "@/lib/moderation/policies";

export async function POST(req: NextRequest) {
  const { text, policy: policyName = "moderate" } = await req.json();

  if (!text || typeof text !== "string") {
    return NextResponse.json({ error: "Text is required" }, { status: 400 });
  }

  const result = await moderateText(text);
  const policy = policies[policyName] || policies.moderate;
  const policyResult = checkPolicy(policy, result.categories, result.categoryScores);

  return NextResponse.json({
    flagged: result.flagged,
    passed: policyResult.passed,
    action: policyResult.action,
    violations: policyResult.violations,
    scores: result.categoryScores,
  });
}

"use client";

import { useState, useCallback } from "react";

export function useModeration(policy = "moderate") {
  const [isChecking, setIsChecking] = useState(false);
  const [result, setResult] = useState<{
    passed: boolean;
    violations: string[];
  } | null>(null);

  const checkContent = useCallback(
    async (text: string) => {
      setIsChecking(true);
      try {
        const response = await fetch("/api/moderate", {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ text, policy }),
        });
        const data = await response.json();
        setResult(data);
        return data;
      } finally {
        setIsChecking(false);
      }
    },
    [policy]
  );

  return { isChecking, result, checkContent };
}