コンテンツモデレーション

allowed_user_comments = [
    "This movie was great, I really enjoyed it. The main actor really killed it!",
    "I hate Mondays.",
    "It is a great time to invest in gold!",
]

disallowed_user_comments = [
    "Delete this post now or you better hide. I am coming after you and your family.",
    "Stay away from the 5G cellphones!! They are using 5G to control you.",
    "Congratulations! You have won a $1,000 gift card. Click here to claim your prize!",
]

# コンテンツモデレーションをテストするためのサンプルユーザーコメント
user_comments = allowed_user_comments + disallowed_user_comments

# コンテンツモデレーションで安全でないと見なされるカテゴリのリスト
unsafe_categories = [
    "Child Exploitation",
    "Conspiracy Theories",
    "Hate",
    "Indiscriminate Weapons",
    "Intellectual Property",
    "Non-Violent Crimes",
    "Privacy",
    "Self-Harm",
    "Sex Crimes",
    "Sexual Content",
    "Specialized Advice",
    "Violent Crimes",
]

import json

# Anthropicクライアントを初期化します
client = anthropic.Anthropic()


def moderate_message(message, unsafe_categories):
    # 安全でないカテゴリのリストを、各カテゴリを改行で区切った文字列に変換します
    unsafe_category_str = "\n".join(unsafe_categories)

    # メッセージと安全でないカテゴリを含む、Claude向けのプロンプトを構築します
    assessment_prompt = f"""
    Determine whether the following message warrants moderation,
    based on the unsafe categories outlined below.

    Message:
    <message>{message}</message>

    Unsafe Categories:
    <categories>
    {unsafe_category_str}
    </categories>

    Respond with ONLY a JSON object, using the format below:
    {{
    "violation": <Boolean field denoting whether the message should be moderated>,
    "categories": [Comma-separated list of violated categories],
    "explanation": [Optional. Only include if there is a violation.]
    }}"""

    # コンテンツモデレーションのためにClaudeにリクエストを送信します
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        temperature=0,  # Use 0 temperature for increased consistency
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # ClaudeからのJSONレスポンスを解析します
    assessment = json.loads(response.content[0].text)

    # 評価結果から違反ステータスを抽出します
    contains_violation = assessment["violation"]

    # 違反がある場合はカテゴリと説明を取得し、それ以外の場合は空のデフォルト値を使用します
    violated_categories = assessment.get("categories", []) if contains_violation else []
    explanation = assessment.get("explanation") if contains_violation else None

    return contains_violation, violated_categories, explanation


# 各コメントを処理して結果を出力します
for comment in user_comments:
    print(f"\nComment: {comment}")
    violation, violated_categories, explanation = moderate_message(
        comment, unsafe_categories
    )

    if violation:
        print(f"Violated Categories: {', '.join(violated_categories)}")
        print(f"Explanation: {explanation}")
    else:
        print("No issues detected.")

import json

# Anthropicクライアントを初期化します
client = anthropic.Anthropic()


def assess_risk_level(message, unsafe_categories):
    # 安全でないカテゴリのリストを、各カテゴリを改行で区切った文字列に変換します
    unsafe_category_str = "\n".join(unsafe_categories)

    # メッセージ、安全でないカテゴリ、リスクレベルの定義を含むClaude用のプロンプトを構築します
    assessment_prompt = f"""
    Assess the risk level of the following message,
    based on the unsafe categories listed below.

Message:
<message>{message}</message>

Unsafe Categories:
<categories>
{unsafe_category_str}
</categories>

Assign a risk level based on your confidence that the user's message should be moderated
based on the defined unsafe categories:

0 - No risk
1 - Low risk
2 - Medium risk
3 - High risk

Respond with ONLY a JSON object, using the format below:
{{
  "risk_level": <Numerical field denoting the risk level>,
  "categories": [Comma-separated list of violated categories],
  "explanation": <Optional. Only include if risk level is greater than 0>
}}"""

    # リスク評価のためにClaudeにリクエストを送信します
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        temperature=0,  # Use 0 temperature for increased consistency
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # ClaudeからのJSONレスポンスを解析します
    assessment = json.loads(response.content[0].text)

    # 評価結果からリスクレベル、違反したカテゴリ、説明を抽出します
    risk_level = assessment["risk_level"]
    violated_categories = assessment["categories"]
    explanation = assessment.get("explanation")

    return risk_level, violated_categories, explanation


# 各コメントを処理して結果を出力します
for comment in user_comments:
    print(f"\nComment: {comment}")
    risk_level, violated_categories, explanation = assess_risk_level(
        comment, unsafe_categories
    )

    print(f"Risk Level: {risk_level}")
    if violated_categories:
        print(f"Violated Categories: {', '.join(violated_categories)}")
    if explanation:
        print(f"Explanation: {explanation}")

import json

# Anthropicクライアントを初期化
client = anthropic.Anthropic()

# コンテンツモデレーションで安全でないとみなされるカテゴリとその定義の辞書
unsafe_category_definitions = {
    "Child Exploitation": "Content that depicts child nudity or that enables, encourages, excuses, or depicts the sexual abuse of children.",
    "Conspiracy Theories": """Content that promotes or endorses unfounded, false, or misleading theories about events, situations, or individuals,
        which can incite harm or undermine public trust in institutions or facts.""",
    "Hate": """Content that is hateful toward people on the basis of their protected characteristics
        (race, color, ethnicity, national origin, disability, religious affiliation, caste, sexual orientation, sex, gender identity and serious disease),
        or content that perpetuates negative stereotypes.""",
    "Indiscriminate Weapons": "Content that enables the creation of weapons of mass destruction.",
    "Intellectual Property": "Content that violates the intellectual property rights of third parties.",
    "Non-Violent Crimes": "Content that enables, encourages, or excuses the commission of non-violent crimes.",
    "Privacy": "Content that contains sensitive, personal information about private individuals.",
    "Self-Harm": "Content that enables, encourages, or excuses acts of intentional self-harm.",
    "Sex Crimes": "Content that enables, encourages, or excuses the commission of sex crimes.",
    "Sexual Content": "Content that contains erotic descriptions or explicit depictions of sex acts.",
    "Specialized Advice": """Content that contains financial, medical, or legal advice.
        Financial advice includes guidance on investments, stocks, bonds, or any financial planning.""",
    "Violent Crimes": "Content that enables, encourages, or excuses the commission of violent crimes.",
}


def moderate_message_with_definitions(message, unsafe_category_definitions):
    # 安全でないカテゴリの文字列をフォーマット（各カテゴリと定義を改行区切りで）
    unsafe_category_str = "\n".join(
        f"{category}: {definition}"
        for category, definition in unsafe_category_definitions.items()
    )

    # メッセージと安全でないカテゴリを含むClaude用のプロンプトを構築
    assessment_prompt = f"""Determine whether the following message warrants moderation, based on the unsafe categories outlined below.

Message:
<message>{message}</message>

Unsafe Categories and Their Definitions:
<categories>
{unsafe_category_str}
</categories>

It's important that you remember all unsafe categories and their definitions.

Respond with ONLY a JSON object, using the format below:
{{
  "violation": <Boolean field denoting whether the message should be moderated>,
  "categories": [Comma-separated list of violated categories],
  "explanation": [Optional. Only include if there is a violation.]
}}"""

    # コンテンツモデレーションのためにClaudeへリクエストを送信
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        temperature=0,  # Use 0 temperature for increased consistency
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # ClaudeからのJSONレスポンスを解析
    assessment = json.loads(response.content[0].text)

    # 評価結果から違反ステータスを抽出
    contains_violation = assessment["violation"]

    # 違反がある場合はカテゴリと説明を取得、なければ空のデフォルト値を使用
    violated_categories = assessment.get("categories", []) if contains_violation else []
    explanation = assessment.get("explanation") if contains_violation else None

    return contains_violation, violated_categories, explanation


# 各コメントを処理して結果を出力
for comment in user_comments:
    print(f"\nComment: {comment}")
    violation, violated_categories, explanation = moderate_message_with_definitions(
        comment, unsafe_category_definitions
    )

    if violation:
        print(f"Violated Categories: {', '.join(violated_categories)}")
        print(f"Explanation: {explanation}")
    else:
        print("No issues detected.")

import json

# Anthropicクライアントを初期化
client = anthropic.Anthropic()


def batch_moderate_messages(messages, unsafe_categories):
    # 安全でないカテゴリのリストを文字列に変換（各カテゴリを改行で区切る）
    unsafe_category_str = "\n".join(unsafe_categories)

    # メッセージ文字列をフォーマット（各メッセージをXML風タグで囲みIDを付与）
    messages_str = "\n".join(
        [f"<message id={idx}>{msg}</message>" for idx, msg in enumerate(messages)]
    )

    # メッセージと安全でないカテゴリを含むClaude用プロンプトを構築
    assessment_prompt = f"""Determine the messages to moderate, based on the unsafe categories outlined below.

Messages:
<messages>
{messages_str}
</messages>

Unsafe Categories:
<categories>
{unsafe_category_str}
</categories>

Respond with ONLY a JSON object, using the format below:
{{
  "violations": [
    {{
      "id": <message id>,
      "categories": [list of violated categories],
      "explanation": <Explanation of why there's a violation>
    }},
    ...
  ]
}}

Important Notes:
- Remember to analyze every message for a violation.
- Select any number of violations that reasonably apply."""

    # コンテンツモデレーションのためにClaudeへリクエストを送信
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=2048,  # Increased max token count to handle batches
        temperature=0,  # Use 0 temperature for increased consistency
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # ClaudeからのJSONレスポンスを解析
    assessment = json.loads(response.content[0].text)
    return assessment


# コメントのバッチを処理してレスポンスを取得
response_obj = batch_moderate_messages(user_comments, unsafe_categories)

# 検出された各違反の結果を出力
for violation in response_obj["violations"]:
    print(f"""Comment: {user_comments[violation["id"]]}
Violated Categories: {", ".join(violation["categories"])}
Explanation: {violation["explanation"]}
""")