New and Improved Content Moderation Tooling

To help developers protect their applications against possible misuse, we are introducing the faster and more accurate Moderation endpoint. This endpoint provides OpenAI API developers with free access to GPT-based classifiers that detect undesired content—an instance of using AI systems to assist with human supervision of these systems. We have also released both a technical paper describing our methodology and the dataset used for evaluation.

When given a text input, the Moderation endpoint assesses whether the content is sexual, hateful, violent, or promotes self-harm—content prohibited by our content policy. The endpoint has been trained to be quick, accurate, and to perform robustly across a range of applications. Importantly, this reduces the chances of products “saying” the wrong thing, even when deployed to users at-scale. As a consequence, AI can unlock benefits in sensitive settings, like education, where it could not otherwise be used with confidence.

The Moderation endpoint helps developers to benefit from our infrastructure investments. Rather than build and maintain their own classifiers—an extensive process, as we document in our paper—they can instead access accurate classifiers through a single API call.

As part of OpenAI’s commitment to making the AI ecosystem safer, we are providing this endpoint to allow free moderation of all OpenAI API-generated content. For instance, Inworld, an OpenAI API customer, uses the Moderation endpoint to help their AI-based virtual characters remain appropriate for their audiences. By leveraging OpenAI’s technology, Inworld can focus on their core product: creating memorable characters. We currently do not support monitoring of third-party traffic.

---

Get started with the Moderation endpoint by checking out the documentation. More details of the training process and model performance are available in our paper. We have also released an evaluation dataset, featuring Common Crawl data labeled within these categories, which we hope will spur further research in this area.

.animation {
width: 100%;
max-width: 900px;
margin: 0 auto;
display: grid;
grid-template-rows: repeat(2, auto);
font-family: ColfaxAI, Helvetica, sans-serif;
font-size: 16px;
gap: 120px 0;
position: relative;
}

@media (min-width: 900px) {
.animation {
grid-template-rows: auto;
grid-template-columns: repeat(2, 1fr);
}
}

.animation-endpoint {
aspect-ratio: 1.7 / 1;
border: 1.5px solid #000;
border-radius: 8px;
text-align: center;
position: relative;
}

.animation-inputWrapper {
display: flex;
align-items: center;
min-height: 80px;
}

.animation-input {
position: relative;
border-radius: 8px;
width: 150px;
padding: 12px;
opacity: 0;
}

.animation-inputBorder {
position: absolute;
top: 0;
left: 0;
fill: none;
}

.animation-inputLabel {
display: block;
position: relative;
top: -5px;
margin-bottom: -2px;
}

.animation-inputLinesWrapper {
display: grid;
grid-auto-flow: row;
grid-auto-rows: 6px;
gap: 4px 0;
position: relative;
}

.animation-inputLine {
background-color: #E5E5E5;
border-radius: 4px;
}

.animation-inputLine:nth-child(2) { max-width: 85%; }
.animation-inputLine:nth-child(3) { max-width: 90%; }
.animation-inputLine:nth-child(4) { max-width: 70%; }

.animation-endpoint-name {
position: absolute;
line-height: 1;
top: 0;
left: 0;
transform: translateY(-100%) translateY(-12px);
}

.animation-endpoint-internals {
padding: 12px;
gap: 12px 12px;
display: grid;
grid-template-columns: repeat(2, 1fr);
height: 100%;
position: relative;
}

@media (min-width: 900px) {
.animation-endpoint-internals {
grid-template-columns: repeat(4, 1fr);
}
}

.animation-endpoint-check-type {
border: 1.5px solid #000;
border-radius: 6px;
display: flex;
align-items: center;
justify-content: center;
opacity: 0.2;
transition: all 350ms cubic-bezier(.645, .045, .355, 1);
}

.animation-endpoint-check-type.flagged {
border-color: #ff6e3c;
color: #ff6e3c;
}

.animation-arrow {
position: absolute;
opacity: 0;
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
text-align: center;
}

.animation-arrow–down {
top: 107px;
left: 20px;
}

.animation-arrow–up {
top: 107px;
left: 35px;
}

.animation-arrow–left,
.animation-arrow–right {
display: none;
}

.animation-arrow–left,
.animation-arrow–right {
left: 27%;
top: calc(50% – 5px);
flex-direction: column;
}

.animation-arrow–left {
top: calc(50% – 5px + 8px);
}

.animation-arrow–right {
top: calc(50% – 5px – 8px);
}

@media (max-width: 899px) {
.animation-arrow span {
position: relative;
top: 4px;
margin-left: 8px;
}
}

@media (min-width: 900px) {
.animation-arrow–up,
.animation-arrow–down {
display: none;
}

.animation-arrow–left,
.animation-arrow–right {
display: flex;
}
}

const MOBILE_QUERY = “(max-width: 600px)”;
const STROKE_LENGTH = 441.259521484375;

let isMobile = window.matchMedia(MOBILE_QUERY).matches;
let lastResultFlagged = false;
let lastFlaggedCategoryIndex = 0;

const animation = document.querySelector(“.js-animation”);
const input = document.querySelector(‘.js-input’);
const [downArrow, upArrow, leftArrow, rightArrow] = […document.querySelectorAll(‘.js-arrow’)];
const [defaultOutline, badOutline, goodOutline] = […document.querySelectorAll(‘.js-input-outline’)];
const endPointInternals = […document.querySelectorAll(‘.js-endpoint-internals’)];
const [mobileResultLabel, desktopResultLabel] = […document.querySelectorAll(‘.js-result’)];

const handleResize = () => {
isMobile = window.matchMedia(MOBILE_QUERY).matches;
};

// Animation methods
const prepare = () => new Promise((resolve) => {
[defaultOutline, goodOutline, badOutline].forEach((el, index) => {
el.getAnimations({ subtree: true }).forEach((animation) => {
animation.cancel();
});

console.log(el.getAnimations({subtree: true}));
el.style.strokeDashoffset = index > 0 ? STROKE_LENGTH : 0;
});

resolve();
});

const animateInputIn = () => new Promise((resolve, reject) => {
const keyframes = isMobile ? [
{transform: ‘scale(0.85)’, opacity: 0},
{opacity: 1, offset: 0.4},
{transform: ‘scale(1.0)’, opacity: 1},
] : [
{transform: ‘translateY(-30px) scale(0.85)’, opacity: 0},
{opacity: 1, offset: 0.4},
{transform: ‘translateY(0) scale(1)’, opacity: 1},
];

const animation = input.animate(
keyframes,
{fill: ‘forwards’, duration: 500, easing: ‘cubic-bezier(.215, .61, .355, 1)’}
);

animation.onfinish = () => {
resolve();
}
});

const animateRequest = () => new Promise((resolve, reject) => {
const element = isMobile ? downArrow : leftArrow;
const keyframes = isMobile ? [
{transform: ‘translateY(-10px)’, opacity: 0},
{transform: ‘translateX(0)’, opacity: 1}
] : [
{transform: ‘translateX(-20px)’, opacity: 0},
{transform: ‘translateX(0)’, opacity: 1}
];

const animation = element.animate(
keyframes,
{fill: ‘forwards’, duration: 300, easing: ‘cubic-bezier(.215, .61, .355, 1)’}
);

animation.onfinish = () => {
resolve();
}
});

const animateInputOut = () => new Promise((resolve) => {
const keyframes = isMobile ? [
{transform: ‘scale(1.0)’, opacity: 1},
{transform: ‘scale(0.85)’, opacity: 0},
] : [
{transform: ‘translateY(0) scale(1)’, opacity: 1},
{opacity: 0, offset: 0.4},
{transform: ‘translateY(30px) scale(0.85)’, opacity: 0}
]
const animation = input.animate(
keyframes,
{fill: ‘forwards’, duration: 1000, easing: ‘cubic-bezier(.215, .61, .355, 1)’}
);

animation.onfinish = () => {
resolve();
}
});

const animateInternals = (flag) => {
let flaggedFactorIndex = lastFlaggedCategoryIndex;

if (flag) {
while (flaggedFactorIndex === lastFlaggedCategoryIndex) {
flaggedFactorIndex = Math.floor(Math.random() * endPointInternals.length);
}

lastFlaggedCategoryIndex = flaggedFactorIndex;
}

return Promise.all(endPointInternals.map((el, index) => new Promise((resolve, reject) => {
const isFlagged = flag && index === flaggedFactorIndex;
const keyframes = isFlagged
? [{opacity: 0.2}, {opacity: 1.0}, {opacity: 0.8}, {opacity: 1.0}]
: [{opacity: 0.2}, {opacity: 1.0}, {opacity: 0.8}, {opacity: 1.0}, {opacity: 0.2}];
const animation = el.animate(
keyframes,
{duration: isFlagged ? 800 : 1000, delay: index * 150, easing: ‘cubic-bezier(.645, .045, .355, 1)’, fill: ‘forwards’},
);

if (isFlagged) {
setTimeout(() => {
endPointInternals[index].classList.add(‘flagged’);
}, 200);
}

animation.onfinish = () => {
resolve();
}
})));
}

const animateResponse = (flag) => new Promise((resolve, reject) => {
const element = isMobile ? upArrow : rightArrow;
const resultLabel = isMobile ? mobileResultLabel : desktopResultLabel;

if (flag) {
resultLabel.innerText = ‘flagged’ ;
resultLabel.style.color = ‘#FF6E3C’;
} else {
resultLabel.innerText = ‘not flagged’ ;
resultLabel.style.color = ‘#51DA4C’;
}

const keyframes = isMobile ? [
{transform: ‘translateY(10px)’, opacity: 0},
{transform: ‘translateY(0)’, opacity: 1}
] : [
{transform: ‘translateX(20px)’, opacity: 0},
{transform: ‘translateX(0)’, opacity: 1}
]

const animation = element.animate(
keyframes,
{fill: ‘forwards’, duration: 300, easing: ‘cubic-bezier(.215, .61, .355, 1)’}
);

animation.onfinish = () => {
resolve();
}
});

const animateResult = (flag) => new Promise((resolve, reject) => {
const animateOutDefault = defaultOutline.animate(
[{strokeDashoffset: 0}, {strokeDashoffset: -1 * STROKE_LENGTH}],
{fill: ‘forwards’, duration: 1400, easing: ‘cubic-bezier(.645, .045, .355, 1)’}
);

const resultEl = flag ? badOutline : goodOutline;
const animateInResult = resultEl.animate(
[{strokeDashoffset: STROKE_LENGTH}, {strokeDashoffset: 0}],
{fill: ‘forwards’, duration: 1400, easing: ‘cubic-bezier(.645, .045, .355, 1)’}
);

animateInResult.onfinish = () => {
resolve();
}
});

const animateArrowsOut = () => new Promise((resolveAll) => {
Promise.all([leftArrow, rightArrow, upArrow, downArrow].map((el) => new Promise((resolve) => {
const animation = el.animate(
[{opacity: 1}, {opacity: 0}],
{duration: 500, delay: 1500, fill: ‘forwards’, easing: ‘cubic-bezier(.215, .61, .355, 1)’}
);

animation.onfinish = () => {
resolve();
}
}))).then(() => {
resolveAll();
});
});

const resetInternals = () => new Promise((resolve) => {
endPointInternals.forEach((factor) => {
const isFlagged = factor.classList.contains(‘flagged’);

if (isFlagged) {
factor.classList.remove(‘flagged’);

factor.animate(
[{opacity: 1.0}, {opacity: 0.2}],
{duration: 500, easing: ‘cubic-bezier(.645, .045, .355, 1)’, fill: ‘forwards’}
)
}
});

resolve();
});

const cleanUp = () => new Promise((resolve) => {
mobileResultLabel.innerText = ”;
desktopResultLabel.innerText = ”;
resolve();
});

// Main logic
const ANIMATION_STEPS = [
prepare,
animateInputIn,
animateRequest,
animateInternals,
animateResponse,
animateResult,
animateArrowsOut,
resetInternals,
animateInputOut,
cleanUp,
]

const animate = async () => {
const flagResult = !lastResultFlagged;

for (const animation of ANIMATION_STEPS) {
await animation(flagResult);
}

lastResultFlagged = flagResult;

setTimeout(() => {
animate();
}, 500)
}

window.addEventListener(‘resize’, handleResize);
window.addEventListener(‘DOMContentLoaded’, () => {
animate();
});