Inhaltsmoderation

client = anthropic.Anthropic()

allowed_user_comments = [
    "This movie was great, I really enjoyed it. The main actor really killed it!",
    "I hate Mondays.",
    "It is a great time to invest in gold!",
]

disallowed_user_comments = [
    "Delete this post now or you better hide. I am coming after you and your family.",
    "Stay away from the 5G cellphones!! They are using 5G to control you.",
    "Congratulations! You have won a $1,000 gift card. Click here to claim your prize!",
]

# Beispiel-Nutzerkommentare zum Testen der Content-Moderation
user_comments = allowed_user_comments + disallowed_user_comments

# Kategorien, die für die Content-Moderation als unsicher gelten
unsafe_categories = [
    "Child Exploitation",
    "Conspiracy Theories",
    "Hate",
    "Indiscriminate Weapons",
    "Intellectual Property",
    "Non-Violent Crimes",
    "Privacy",
    "Self-Harm",
    "Sex Crimes",
    "Sexual Content",
    "Specialized Advice",
    "Violent Crimes",
]

def moderate_message(message, unsafe_categories):
    # Wandle die unsicheren Kategorien in einen String um, mit jeder Kategorie in einer neuen Zeile
    unsafe_category_str = "\n".join(unsafe_categories)

    # Erstelle den Prompt für Claude, einschließlich der Nachricht und der unsicheren Kategorien
    assessment_prompt = f"""
    Determine whether the following message warrants moderation,
    based on the unsafe categories outlined below.

    Message:
    <message>{message}</message>

    Unsafe Categories:
    <categories>
    {unsafe_category_str}
    </categories>

    Respond with ONLY a JSON object, using the format below:
    {{
    "violation": <Boolean field denoting whether the message should be moderated>,
    "categories": [Comma-separated list of violated categories],
    "explanation": [Optional. Only include if there is a violation.]
    }}
Do not include markdown formatting or code fences in your response."""

    # Sende die Anfrage zur Content-Moderation an Claude
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # Parse die JSON-Antwort von Claude
    text_block = next(block for block in response.content if block.type == "text")
    assessment = json.loads(text_block.text)

    # Extrahiere den Verstoß-Status aus der Bewertung
    contains_violation = assessment["violation"]

    # Bei einem Verstoß hole die Kategorien und die Erklärung; andernfalls verwende leere Standardwerte
    violated_categories = assessment.get("categories", []) if contains_violation else []
    explanation = assessment.get("explanation") if contains_violation else None

    return contains_violation, violated_categories, explanation


# Verarbeite jeden Kommentar und gib die Ergebnisse aus
for comment in user_comments:
    print(f"\nComment: {comment}")
    violation, violated_categories, explanation = moderate_message(
        comment, unsafe_categories
    )

    if violation:
        print(f"Violated Categories: {', '.join(violated_categories)}")
        print(f"Explanation: {explanation}")
    else:
        print("No issues detected.")

def assess_risk_level(message, unsafe_categories):
    # Wandle die unsicheren Kategorien in einen String um, mit jeder Kategorie in einer neuen Zeile
    unsafe_category_str = "\n".join(unsafe_categories)

    # Erstelle den Prompt für Claude, einschließlich der Nachricht, der unsicheren Kategorien und der Risikostufen-Definitionen
    assessment_prompt = f"""
    Assess the risk level of the following message,
    based on the unsafe categories listed below.

Message:
<message>{message}</message>

Unsafe Categories:
<categories>
{unsafe_category_str}
</categories>

Assign a risk level based on your confidence that the user's message should be moderated
based on the defined unsafe categories:

0 - No risk
1 - Low risk
2 - Medium risk
3 - High risk

Respond with ONLY a JSON object, using the format below:
{{
  "risk_level": <Numerical field denoting the risk level>,
  "categories": [Comma-separated list of violated categories],
  "explanation": <Optional. Only include if risk level is greater than 0>
}}
Do not include markdown formatting or code fences in your response."""

    # Sende die Anfrage zur Risikobewertung an Claude
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # Parse die JSON-Antwort von Claude
    text_block = next(block for block in response.content if block.type == "text")
    assessment = json.loads(text_block.text)

    # Extrahiere die Risikostufe, die verletzten Kategorien und die Erklärung aus der Bewertung
    risk_level = assessment["risk_level"]
    violated_categories = assessment["categories"]
    explanation = assessment.get("explanation")

    return risk_level, violated_categories, explanation


# Verarbeite jeden Kommentar und gib die Ergebnisse aus
for comment in user_comments:
    print(f"\nComment: {comment}")
    risk_level, violated_categories, explanation = assess_risk_level(
        comment, unsafe_categories
    )

    print(f"Risk Level: {risk_level}")
    if violated_categories:
        print(f"Violated Categories: {', '.join(violated_categories)}")
    if explanation:
        print(f"Explanation: {explanation}")

# Kategorien, die für die Content-Moderation als unsicher gelten, mit ihren Definitionen
unsafe_category_definitions = {
    "Child Exploitation": "Content that depicts child nudity or that enables, encourages, excuses, or depicts the sexual abuse of children.",
    "Conspiracy Theories": """Content that promotes or endorses unfounded, false, or misleading theories about events, situations, or individuals,
        which can incite harm or undermine public trust in institutions or facts.""",
    "Hate": """Content that is hateful toward people on the basis of their protected characteristics
        (race, color, ethnicity, national origin, disability, religious affiliation, caste, sexual orientation, sex, gender identity and serious disease),
        or content that perpetuates negative stereotypes.""",
    "Indiscriminate Weapons": "Content that enables the creation of weapons of mass destruction.",
    "Intellectual Property": "Content that violates the intellectual property rights of third parties.",
    "Non-Violent Crimes": "Content that enables, encourages, or excuses the commission of non-violent crimes.",
    "Privacy": "Content that contains sensitive, personal information about private individuals.",
    "Self-Harm": "Content that enables, encourages, or excuses acts of intentional self-harm.",
    "Sex Crimes": "Content that enables, encourages, or excuses the commission of sex crimes.",
    "Sexual Content": "Content that contains erotic descriptions or explicit depictions of sex acts.",
    "Specialized Advice": """Content that contains financial, medical, or legal advice.
        Financial advice includes guidance on investments, stocks, bonds, or any financial planning.""",
    "Violent Crimes": "Content that enables, encourages, or excuses the commission of violent crimes.",
}


def moderate_message_with_definitions(message, unsafe_category_definitions):
    # Formatiere den String der unsicheren Kategorien, mit jeder Kategorie und ihrer Definition in einer neuen Zeile
    unsafe_category_str = "\n".join(
        f"{category}: {definition}"
        for category, definition in unsafe_category_definitions.items()
    )

    # Erstelle den Prompt für Claude, einschließlich der Nachricht und der unsicheren Kategorien
    assessment_prompt = f"""Determine whether the following message warrants moderation, based on the unsafe categories outlined below.

Message:
<message>{message}</message>

Unsafe Categories and Their Definitions:
<categories>
{unsafe_category_str}
</categories>

It's important that you remember all unsafe categories and their definitions.

Respond with ONLY a JSON object, using the format below:
{{
  "violation": <Boolean field denoting whether the message should be moderated>,
  "categories": [Comma-separated list of violated categories],
  "explanation": [Optional. Only include if there is a violation.]
}}
Do not include markdown formatting or code fences in your response."""

    # Sende die Anfrage zur Content-Moderation an Claude
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=200,
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # Parse die JSON-Antwort von Claude
    text_block = next(block for block in response.content if block.type == "text")
    assessment = json.loads(text_block.text)

    # Extrahiere den Verstoß-Status aus der Bewertung
    contains_violation = assessment["violation"]

    # Bei einem Verstoß hole die Kategorien und die Erklärung; andernfalls verwende leere Standardwerte
    violated_categories = assessment.get("categories", []) if contains_violation else []
    explanation = assessment.get("explanation") if contains_violation else None

    return contains_violation, violated_categories, explanation


# Verarbeite jeden Kommentar und gib die Ergebnisse aus
for comment in user_comments:
    print(f"\nComment: {comment}")
    violation, violated_categories, explanation = moderate_message_with_definitions(
        comment, unsafe_category_definitions
    )

    if violation:
        print(f"Violated Categories: {', '.join(violated_categories)}")
        print(f"Explanation: {explanation}")
    else:
        print("No issues detected.")

def batch_moderate_messages(messages, unsafe_categories):
    # Wandle die unsicheren Kategorien in einen String um, mit jeder Kategorie in einer neuen Zeile
    unsafe_category_str = "\n".join(unsafe_categories)

    # Formatiere den Nachrichten-String, wobei jede Nachricht in XML-ähnliche Tags eingeschlossen wird und eine ID erhält
    messages_str = "\n".join(
        [f"<message id={idx}>{msg}</message>" for idx, msg in enumerate(messages)]
    )

    # Erstelle den Prompt für Claude, einschließlich der Nachrichten und der unsicheren Kategorien
    assessment_prompt = f"""Determine the messages to moderate, based on the unsafe categories outlined below.

Messages:
<messages>
{messages_str}
</messages>

Unsafe Categories:
<categories>
{unsafe_category_str}
</categories>

Respond with ONLY a JSON object, using the format below:
{{
  "violations": [
    {{
      "id": <message id>,
      "categories": [list of violated categories],
      "explanation": <Explanation of why there's a violation>
    }}
  ]
}}

Important Notes:
- Remember to analyze every message for a violation.
- Select any number of violations that reasonably apply.
- Do not include markdown formatting or code fences in your response."""

    # Sende die Anfrage zur Content-Moderation an Claude
    response = client.messages.create(
        model="claude-haiku-4-5-20251001",  # Using the Haiku model for lower costs
        max_tokens=2048,  # Increased max token count to handle batches
        messages=[{"role": "user", "content": assessment_prompt}],
    )

    # Parse die JSON-Antwort von Claude
    text_block = next(block for block in response.content if block.type == "text")
    assessment = json.loads(text_block.text)
    return assessment


# Verarbeite den Batch von Kommentaren und hole die Antwort
response_obj = batch_moderate_messages(user_comments, unsafe_categories)

# Gib die Ergebnisse für jeden erkannten Verstoß aus
for violation in response_obj["violations"]:
    print(f"""Comment: {user_comments[violation["id"]]}
Violated Categories: {", ".join(violation["categories"])}
Explanation: {violation["explanation"]}
""")