Spring Boot, Google Cloud Vertex AI 및 Gemini 모델을 사용한 이미지 기반 상품 검색

<!-- config version for dependency-->
<properties>
    <spring-cloud-gcp.version>5.1.2</spring-cloud-gcp.version>
    <google-cloud-bom.version>26.32.0</google-cloud-bom.version>
</properties>

<!-- In your dependencyManagement, please add 2 dependencies below -->
<dependencyManagement>
  <dependencies>
      <dependency>
          <groupId>com.google.cloud</groupId>
          <artifactId>spring-cloud-gcp-dependencies</artifactId>
          <version>${spring-cloud-gcp.version}</version>
          <type>pom</type>
          <scope>import</scope>
      </dependency>

      <dependency>
          <groupId>com.google.cloud</groupId>
          <artifactId>libraries-bom</artifactId>
          <version>${google-cloud-bom.version}</version>
          <type>pom</type>
          <scope>import</scope>
      </dependency>
  </dependencies>
</dependencyManagement>

<!-- In your tab dependencies, please add the dependency below -->
<dependencies>
  <dependency>
      <groupId>com.google.cloud</groupId>
      <artifactId>google-cloud-vertexai</artifactId>
  </dependency>
</dependencies>

import com.google.cloud.vertexai.VertexAI;
import com.google.cloud.vertexai.generativeai.GenerativeModel;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration(proxyBeanMethods = false)
public class GeminiConfig {

    private static final String MODEL_NAME = "gemini-1.5-flash";
    private static final String LOCATION = "asia-southeast1";
    private static final String PROJECT_ID = "yasmini";

    @Bean
    public VertexAI vertexAI() {
        return new VertexAI(PROJECT_ID, LOCATION);
    }

    @Bean
    public GenerativeModel getModel(VertexAI vertexAI) {
        return new GenerativeModel(MODEL_NAME, vertexAI);
    }
}

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.cloud.vertexai.api.Content;
import com.google.cloud.vertexai.api.GenerateContentResponse;
import com.google.cloud.vertexai.api.Part;
import com.google.cloud.vertexai.generativeai.*;
import com.learning.yasminishop.common.entity.Product;
import com.learning.yasminishop.common.exception.AppException;
import com.learning.yasminishop.common.exception.ErrorCode;
import com.learning.yasminishop.product.ProductRepository;
import com.learning.yasminishop.product.dto.response.ProductResponse;
import com.learning.yasminishop.product.mapper.ProductMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;

import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;

@Service
@RequiredArgsConstructor
@Slf4j
@Transactional(readOnly = true)
public class YasMiniAIService {

    private final GenerativeModel generativeModel;
    private final ProductRepository productRepository;

    private final ProductMapper productMapper;


    public List<ProductResponse> findCarByImage(MultipartFile file){
        try {
            var prompt = "Extract the name car to a list keyword and output them in JSON. If you don't find any information about the car, please output the list empty.\nExample response: [\"rolls\", \"royce\", \"wraith\"]";
            var content = this.generativeModel.generateContent(
                    ContentMaker.fromMultiModalData(
                            PartMaker.fromMimeTypeAndData(Objects.requireNonNull(file.getContentType()), file.getBytes()),
                            prompt
                    )
            );

            String jsonContent = ResponseHandler.getText(content);
            log.info("Extracted keywords from image: {}", jsonContent);
            List<String> keywords = convertJsonToList(jsonContent).stream()
                    .map(String::toLowerCase)
                    .toList();

            Set<Product> results = new HashSet<>();
            for (String keyword : keywords) {
                List<Product> products = productRepository.searchByKeyword(keyword);
                results.addAll(products);
            }

            return results.stream()
                    .map(productMapper::toProductResponse)
                    .toList();

        } catch (Exception e) {
            log.error("Error finding car by image", e);
            return List.of();
        }
    }

    private List<String> convertJsonToList(String markdown) throws JsonProcessingException {
        ObjectMapper objectMapper = new ObjectMapper();
        String parseJson = markdown;
        if(markdown.contains("```

json")){
            parseJson = extractJsonFromMarkdown(markdown);
        }
        return objectMapper.readValue(parseJson, List.class);
    }

    private String extractJsonFromMarkdown(String markdown) {
        return markdown.replace("

```json\n", "").replace("\n```

", "");
    }
}

import com.learning.yasminishop.product.dto.response.ProductResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import java.util.List;

@RestController
@RequestMapping("/ai")
@RequiredArgsConstructor
@Slf4j
public class YasMiniAIController {

    private final YasMiniAIService yasMiniAIService;


    @PostMapping
    public List<ProductResponse> findCar(@RequestParam("file") MultipartFile file) {

        var response = yasMiniAIService.findCarByImage(file);
        return response;
    }
}

gcloud auth login