Add OpenAI as a Provider for Descriptive Text Generation

hookdocs/meta.json

@@ -7,5 +7,8 @@
     },
     "migration-guide-v2-to-v3": {
         "title": "Migration guide (version 2 to version 3)"
+    },
+    "prompt-examples": {
+        "title": "Prompt examples"
     }
 }

hookdocs/prompt-examples.md

@@ -0,0 +1,19 @@
+ClassifAI comes with multiple Features that utilize generative AI. These Features use prompts to generate content. ClassifAI ships with default prompts that have been crafted to work for a wide variety of use cases. However, these prompts may not work best for your use case. To help with this, ClassifAI allows you to create custom prompts that you can tailor to your specific needs.
+
+The following are some example prompts for specific Features and use cases that can be used in place of the default prompts that ship with ClassifAI.
+
+## Image Processing
+
+### Descriptive Text Generator
+
+Generate just alt text for an image:
+
+> You are an assistant that generates alt text for images that are used on a website. Alternative (Alt) Text is a brief text description of an image, and is an essential part of making content accessible. Alt Text is read aloud by screen readers to provide information about the image to the reader. You will be provided with an image and will generate alt text for that image. Try and follow the following guidelines when generating Alt Text: 1. Keep it short, usually 1-2 sentences. 2. Consider key elements of the image, instead of describing every little detail. 3. No need to say "image of" or "picture of". Before returning the text, re-evaluate your response and ensure you're following the above points, in particular ensuring the alt text is short, 1-2 sentences max.
+
+Generate just an image caption:
+
+> You are an assistant that generates captions for images that are used on a website. You will be provided with an image and will generate a caption for that image. Please describe the main item you see in the image, giving details but staying concise. There is no need to say "the image contains" or similar, just describe what is actually in the image.
+
+Generate just an image description:
+
+> You are an assistant that generates descriptions of images that are used on a website. You will be provided with an image and will describe the main item you see in the image, giving details but staying concise. There is no need to say "the image contains" or similar, just describe what is actually in the image. Ensure it is descriptive and accurate but not overly verbose.

includes/Classifai/Features/DescriptiveTextGenerator.php

@@ -3,6 +3,7 @@
 namespace Classifai\Features;
 
 use Classifai\Providers\Azure\ComputerVision;
+use Classifai\Providers\OpenAI\ChatGPT;
 use Classifai\Services\ImageProcessing;
 use WP_REST_Server;
 use WP_REST_Request;
@@ -21,6 +22,13 @@ class DescriptiveTextGenerator extends Feature {
 	 */
 	const ID = 'feature_descriptive_text_generator';
 
+	/**
+	 * Prompt for generating descriptive text.
+	 *
+	 * @var string
+	 */
+	public $prompt = 'You are an assistant that generates descriptions of images that are used on a website. You will be provided with an image and will describe the main item you see in the image, giving details but staying concise. There is no need to say "the image contains" or similar, just describe what is actually in the image. This text will be important for screen readers, so make sure it is descriptive and accurate but not overly verbose. Before returning the text, re-evaluate your response and ensure you are following the above points, in particular ensuring the text is concise.';
+
 	/**
 	 * Constructor.
 	 */
@@ -33,6 +41,7 @@ public function __construct() {
 		// Contains just the providers this feature supports.
 		$this->supported_providers = [
 			ComputerVision::ID => __( 'Microsoft Azure AI Vision', 'classifai' ),
+			ChatGPT::ID        => __( 'OpenAI', 'classifai' ),
 		];
 	}
 

includes/Classifai/Providers/OpenAI/ChatGPT.php

@@ -6,6 +6,7 @@
 namespace Classifai\Providers\OpenAI;
 
 use Classifai\Features\ContentResizing;
+use Classifai\Features\DescriptiveTextGenerator;
 use Classifai\Features\ExcerptGeneration;
 use Classifai\Features\TitleGeneration;
 use Classifai\Providers\Provider;
@@ -14,6 +15,8 @@
 
 use function Classifai\get_default_prompt;
 use function Classifai\sanitize_number_of_responses_field;
+use function Classifai\get_modified_image_source_url;
+use function Classifai\get_largest_size_and_dimensions_image_url;
 
 class ChatGPT extends Provider {
 
@@ -132,6 +135,17 @@ public function get_default_provider_settings(): array {
 			case TitleGeneration::ID:
 				$common_settings['number_of_suggestions'] = 1;
 				break;
+
+			case DescriptiveTextGenerator::ID:
+				$common_settings['prompt'] = [
+					[
+						'title'    => esc_html__( 'ClassifAI default', 'classifai' ),
+						'prompt'   => $this->feature_instance->prompt,
+						'original' => 1,
+						'default'  => 1,
+					],
+				];
+				break;
 		}
 
 		return $common_settings;
@@ -189,6 +203,9 @@ public function rest_endpoint_callback( $post_id = 0, string $route_to_call = ''
 
 		// Handle all of our routes.
 		switch ( $route_to_call ) {
+			case 'descriptive_text':
+				$return = $this->generate_descriptive_text( $post_id, $args );
+				break;
 			case 'excerpt':
 				$return = $this->generate_excerpt( $post_id, $args );
 				break;
@@ -203,6 +220,130 @@ public function rest_endpoint_callback( $post_id = 0, string $route_to_call = ''
 		return $return;
 	}
 
+	/**
+	 * Generate descriptive text of an image.
+	 *
+	 * @param int   $post_id The attachment ID we're processing.
+	 * @param array $args Optional arguments.
+	 * @return string|WP_Error
+	 */
+	public function generate_descriptive_text( int $post_id = 0, array $args = [] ) {
+		// Check to be sure the attachment exists and is an image.
+		if ( ! wp_attachment_is_image( $post_id ) ) {
+			return new WP_Error( 'invalid', esc_html__( 'This attachment can\'t be processed.', 'classifai' ) );
+		}
+
+		$metadata = wp_get_attachment_metadata( $post_id );
+
+		if ( ! $metadata || ! is_array( $metadata ) ) {
+			return new WP_Error( 'invalid', esc_html__( 'No valid metadata found.', 'classifai' ) );
+		}
+
+		$image_url = get_modified_image_source_url( $post_id );
+
+		if ( empty( $image_url ) || ! filter_var( $image_url, FILTER_VALIDATE_URL ) ) {
+			if ( isset( $metadata['sizes'] ) && is_array( $metadata['sizes'] ) ) {
+				$image_url = get_largest_size_and_dimensions_image_url(
+					get_attached_file( $post_id ),
+					wp_get_attachment_url( $post_id ),
+					$metadata,
+					[ 512, 2000 ],
+					[ 512, 2000 ],
+					100 * MB_IN_BYTES
+				);
+			} else {
+				$image_url = wp_get_attachment_url( $post_id );
+			}
+		}
+
+		if ( empty( $image_url ) ) {
+			return new WP_Error( 'error', esc_html__( 'Valid image size not found. Make sure the image is bigger than 512x512px.', 'classifai' ) );
+		}
+
+		$feature  = new DescriptiveTextGenerator();
+		$settings = $feature->get_settings();
+
+		// These checks (and the one above) happen in the REST permission_callback,
+		// but we run them again here in case this method is called directly.
+		if ( empty( $settings ) || ( isset( $settings[ static::ID ]['authenticated'] ) && false === $settings[ static::ID ]['authenticated'] ) || ( ! $feature->is_feature_enabled() && ( ! defined( 'WP_CLI' ) || ! WP_CLI ) ) ) {
+			return new WP_Error( 'not_enabled', esc_html__( 'Descriptive text generation is disabled or OpenAI authentication failed. Please check your settings.', 'classifai' ) );
+		}
+
+		$request = new APIRequest( $settings[ static::ID ]['api_key'] ?? '', $feature->get_option_name() );
+
+		/**
+		 * Filter the prompt we will send to ChatGPT.
+		 *
+		 * @since 3.2.0
+		 * @hook classifai_chatgpt_descriptive_text_prompt
+		 *
+		 * @param {string} $prompt Prompt we are sending to ChatGPT.
+		 * @param {int} $post_id ID of attachment we are describing.
+		 *
+		 * @return {string} Prompt.
+		 */
+		$prompt = apply_filters( 'classifai_chatgpt_descriptive_text_prompt', get_default_prompt( $settings[ static::ID ]['prompt'] ?? [] ) ?? $feature->prompt, $post_id );
+
+		/**
+		 * Filter the request body before sending to ChatGPT.
+		 *
+		 * @since 3.2.0
+		 * @hook classifai_chatgpt_descriptive_text_request_body
+		 *
+		 * @param {array} $body Request body that will be sent to ChatGPT.
+		 * @param {int} $post_id ID of attachment we are describing.
+		 *
+		 * @return {array} Request body.
+		 */
+		$body = apply_filters(
+			'classifai_chatgpt_descriptive_text_request_body',
+			[
+				'model'       => $this->chatgpt_model,
+				'messages'    => [
+					[
+						'role'    => 'system',
+						'content' => $prompt,
+					],
+					[
+						'role'    => 'user',
+						'content' => [
+							[
+								'type'      => 'image_url',
+								'image_url' => [
+									'url'    => $image_url,
+									'detail' => 'auto',
+								],
+							],
+						],
+					],
+				],
+				'temperature' => 0.2,
+				'max_tokens'  => 300,
+			],
+			$post_id
+		);
+
+		// Make our API request.
+		$response = $request->post(
+			$this->chatgpt_url,
+			[
+				'body' => wp_json_encode( $body ),
+			]
+		);
+
+		// Extract out the text response, if it exists.
+		if ( ! is_wp_error( $response ) && ! empty( $response['choices'] ) ) {
+			foreach ( $response['choices'] as $choice ) {
+				if ( isset( $choice['message'], $choice['message']['content'] ) ) {
+					// ChatGPT often adds quotes to strings, so remove those as well as extra spaces.
+					$response = sanitize_text_field( trim( $choice['message']['content'], ' "\'' ) );
+				}
+			}
+		}
+
+		return $response;
+	}
+
 	/**
 	 * Generate an excerpt using ChatGPT.
 	 *

includes/Classifai/Services/ImageProcessing.php

@@ -55,6 +55,7 @@ public static function get_service_providers(): array {
 			'classifai_image_processing_service_providers',
 			[
 				'Classifai\Providers\Azure\ComputerVision',
+				'Classifai\Providers\OpenAI\ChatGPT',
 				'Classifai\Providers\OpenAI\DallE',
 			]
 		);

src/js/settings/components/feature-additional-settings/content-resizing.js

@@ -33,7 +33,7 @@ export const ContentResizingSettings = () => {
 			>
 				<PromptRepeater
 					prompts={ featureSettings.condense_text_prompt }
-					setPromts={ ( prompts ) => {
+					setPrompts={ ( prompts ) => {
 						setFeatureSettings( {
 							condense_text_prompt: prompts,
 						} );
@@ -47,7 +47,7 @@ export const ContentResizingSettings = () => {
 			>
 				<PromptRepeater
 					prompts={ featureSettings.expand_text_prompt }
-					setPromts={ ( prompts ) => {
+					setPrompts={ ( prompts ) => {
 						setFeatureSettings( {
 							expand_text_prompt: prompts,
 						} );

src/js/settings/components/feature-additional-settings/excerpt-generation.js

@@ -28,7 +28,7 @@ export const ExcerptGenerationSettings = () => {
 	);
 	const { excerptPostTypes } = window.classifAISettings;
 	const { setFeatureSettings } = useDispatch( STORE_NAME );
-	const setPromts = ( prompts ) => {
+	const setPrompts = ( prompts ) => {
 		setFeatureSettings( {
 			generate_excerpt_prompt: prompts,
 		} );
@@ -45,7 +45,7 @@ export const ExcerptGenerationSettings = () => {
 			>
 				<PromptRepeater
 					prompts={ featureSettings.generate_excerpt_prompt }
-					setPromts={ setPromts }
+					setPrompts={ setPrompts }
 				/>
 			</SettingsRow>
 			<SettingsRow

src/js/settings/components/feature-additional-settings/prompt-repeater.js

@@ -22,14 +22,14 @@ import { __ } from '@wordpress/i18n';
 export const PromptRepeater = ( props ) => {
 	const [ showConfirmDialog, setShowConfirmDialog ] = useState( false );
 	const [ activeIndex, setActiveIndex ] = useState( null );
-	const { prompts = [], setPromts } = props;
+	const { prompts = [], setPrompts } = props;
 
 	const placeholder =
 		prompts?.filter( ( prompt ) => prompt.original )[ 0 ]?.prompt || '';
 
 	// Add a new prompt.
 	const addPrompt = () => {
-		setPromts( [
+		setPrompts( [
 			...prompts,
 			{ default: 0, original: 0, prompt: '', title: '' },
 		] );
@@ -42,7 +42,7 @@ export const PromptRepeater = ( props ) => {
 		if ( prompt[ 0 ]?.default ) {
 			prompts[ 0 ].default = 1;
 		}
-		setPromts( [ ...prompts ] );
+		setPrompts( [ ...prompts ] );
 	};
 
 	// Update prompt.
@@ -60,7 +60,7 @@ export const PromptRepeater = ( props ) => {
 			...prompts[ index ],
 			...changes,
 		};
-		setPromts( [ ...prompts ] );
+		setPrompts( [ ...prompts ] );
 	};
 
 	// Confirm dialog to remove prompt.

src/js/settings/components/feature-additional-settings/title-generation.js

@@ -23,7 +23,7 @@ export const TitleGenerationSettings = () => {
 		select( STORE_NAME ).getFeatureSettings()
 	);
 	const { setFeatureSettings } = useDispatch( STORE_NAME );
-	const setPromts = ( prompts ) => {
+	const setPrompts = ( prompts ) => {
 		setFeatureSettings( {
 			generate_title_prompt: prompts,
 		} );
@@ -39,7 +39,7 @@ export const TitleGenerationSettings = () => {
 		>
 			<PromptRepeater
 				prompts={ featureSettings.generate_title_prompt }
-				setPromts={ setPromts }
+				setPrompts={ setPrompts }
 			/>
 		</SettingsRow>
 	);

src/js/settings/components/provider-settings/openai-chatgpt.js

@@ -12,6 +12,7 @@ import { __ } from '@wordpress/i18n';
 import { SettingsRow } from '../settings-row';
 import { STORE_NAME } from '../../data/store';
 import { useFeatureContext } from '../feature-settings/context';
+import { PromptRepeater } from '../feature-additional-settings/prompt-repeater';
 
 /**
  * Component for OpenAI ChatGPT Provider settings.
@@ -32,6 +33,11 @@ export const OpenAIChatGPTSettings = ( { isConfigured = false } ) => {
 	);
 	const { setProviderSettings } = useDispatch( STORE_NAME );
 	const onChange = ( data ) => setProviderSettings( providerName, data );
+	const setPrompts = ( prompts ) => {
+		setProviderSettings( providerName, {
+			prompt: prompts,
+		} );
+	};
 
 	const Description = () => (
 		<>
@@ -45,6 +51,23 @@ export const OpenAIChatGPTSettings = ( { isConfigured = false } ) => {
 			{ __( 'in order to get your API key.', 'classifai' ) }
 		</>
 	);
+	const promptExamples = (
+		<>
+			{ __( 'Add a custom prompt, if desired. ', 'classifai' ) }
+			{ __( 'See our ', 'classifai' ) }
+			<a
+				href="https://10up.github.io/classifai/tutorial-prompt-examples.html"
+				target="_blank"
+				rel="noopener noreferrer"
+			>
+				{ __( 'documentation', 'classifai' ) }
+			</a>
+			{ __(
+				' for some example prompts you can try that have been tested for specific use cases.',
+				'classifai'
+			) }
+		</>
+	);
 
 	return (
 		<>
@@ -82,6 +105,19 @@ export const OpenAIChatGPTSettings = ( { isConfigured = false } ) => {
 					/>
 				</SettingsRow>
 			) }
+			{ [ 'feature_descriptive_text_generator' ].includes(
+				featureName
+			) && (
+				<SettingsRow
+					label={ __( 'Prompt', 'classifai' ) }
+					description={ promptExamples }
+				>
+					<PromptRepeater
+						prompts={ providerSettings.prompt }
+						setPrompts={ setPrompts }
+					/>
+				</SettingsRow>
+			) }
 		</>
 	);
 };