Skip to content

Rag crud #358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ianardee wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Rag crud #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,40 @@
<artifactId>maven-install-plugin</artifactId>
<version>${org.apache.maven.install.version}</version>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>1.18.20.0</version>
Comment thread
ianardee marked this conversation as resolved.
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>delombok</goal>
</goals>
</execution>
</executions>
<configuration>
<sourceDirectory>${project.basedir}/src/main/java</sourceDirectory>
<outputDirectory>${project.build.directory}/delombok</outputDirectory>
<addOutputDirectory>false</addOutputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${org.apache.maven.javadoc.version}</version>
<configuration>
<sourcepath>${project.build.directory}/delombok</sourcepath>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>${org.apache.maven.site.version}</version>
Expand Down
23 changes: 23 additions & 0 deletions src/main/java/com/mindee/exceptions/MindeeInputException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package com.mindee.exceptions;

import com.mindee.MindeeException;

/**
* Represent an invalid or malformed input.
*/
public class MindeeInputException extends MindeeException {

/**
* {@link Exception}
*/
public MindeeInputException(String message) {
super(message);
}

/**
* {@link Exception}
*/
public MindeeInputException(String message, Exception innerException) {
super(message, innerException);
}
}
244 changes: 238 additions & 6 deletions src/main/java/com/mindee/v2/MindeeClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@
import com.mindee.MindeeException;
import com.mindee.input.LocalInputSource;
import com.mindee.input.URLInputSource;
import com.mindee.v2.clientoptions.BaseAnnotationParameters;
import com.mindee.v2.clientoptions.BaseProductParameters;
import com.mindee.v2.clientoptions.BaseRagDocumentUploadParameters;
import com.mindee.v2.clientoptions.BaseSearchParameters;
import com.mindee.v2.clientoptions.PollingOptions;
import com.mindee.v2.http.MindeeApiV2;
import com.mindee.v2.http.MindeeHttpApiV2;
import com.mindee.v2.http.MindeeHttpExceptionV2;
import com.mindee.v2.parsing.BaseRagAnnotationResponse;
import com.mindee.v2.parsing.BaseResponse;
import com.mindee.v2.parsing.JobResponse;
import com.mindee.v2.parsing.error.ErrorResponse;
Expand Down Expand Up @@ -78,10 +81,10 @@ public JobResponse enqueue(
* Can be used for polling.
*/
public JobResponse getJobFromUrl(String pollingUrl) {
logger.log(System.Logger.Level.INFO, "Getting Job at: {0}", pollingUrl);
if (pollingUrl == null || pollingUrl.isBlank()) {
throw new IllegalArgumentException("Job URL cannot be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Getting Job at: {0}", pollingUrl);
return mindeeApi.reqGetJobByUrl(pollingUrl);
}

Expand All @@ -90,10 +93,10 @@ public JobResponse getJobFromUrl(String pollingUrl) {
* Can be used for polling.
*/
public JobResponse getJob(String jobId) {
logger.log(System.Logger.Level.INFO, "Getting job ID: {0}", jobId);
if (jobId == null || jobId.isBlank()) {
throw new IllegalArgumentException("jobId must not be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Getting job ID: {0}", jobId);
return mindeeApi.reqGetJobById(jobId);
}

Expand All @@ -105,10 +108,10 @@ public <TResponse extends BaseResponse> TResponse getResult(
Class<TResponse> responseClass,
String inferenceId
) {
logger.log(System.Logger.Level.INFO, "Getting result with ID: {0}", inferenceId);
if (inferenceId == null || inferenceId.trim().isEmpty()) {
if (inferenceId == null || inferenceId.isBlank()) {
throw new IllegalArgumentException("inferenceId must not be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Getting result with ID: {0}", inferenceId);
return mindeeApi.reqGetResultById(responseClass, inferenceId);
}

Expand All @@ -120,10 +123,10 @@ public <TResponse extends BaseResponse> TResponse getResultFromUrl(
Class<TResponse> responseClass,
String inferenceUrl
) {
logger.log(System.Logger.Level.INFO, "Getting result at: {0}", inferenceUrl);
if (inferenceUrl == null || inferenceUrl.trim().isEmpty()) {
if (inferenceUrl == null || inferenceUrl.isBlank()) {
throw new IllegalArgumentException("inferenceUrl must not be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Getting result at: {0}", inferenceUrl);
return mindeeApi.reqGetResultByUrl(responseClass, inferenceUrl);
}

Expand Down Expand Up @@ -277,6 +280,235 @@ public SearchResponse searchModels(String modelName, String modelType) {
.reqGetSearch(ModelSearchParameters.builder().name(modelName).modelType(modelType).build());
}

/**
* Not recommended for general use, prefer {@link #uploadAndGetRagDocument}.
* You will need to poll until the document is ready for use.
* Add a document to the RAG database.
*
* @param inputSource The file to upload.
* @param parameters RAG document upload parameters.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse uploadRagDocument(
LocalInputSource inputSource,
BaseRagDocumentUploadParameters<TAnnotationResponse> parameters
) throws IOException {
logger.log(System.Logger.Level.INFO, "Adding a document to the RAG database");
return mindeeApi.reqPostRagDocument(parameters, inputSource);
}

/**
* Add a document to the RAG database, poll, and return the initial annotation.
*
* @param inputSource The file to upload.
* @param parameters RAG document upload parameters.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse uploadAndGetRagDocument(
LocalInputSource inputSource,
BaseRagDocumentUploadParameters<TAnnotationResponse> parameters
) throws IOException, InterruptedException {
return uploadAndGetRagDocument(inputSource, parameters, null);
}

/**
* Add a document to the RAG database, poll, and return the initial annotation.
*
* @param inputSource The file to upload.
* @param parameters RAG document upload parameters.
* @param pollingOptions Polling options (if null, default options are used).
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse uploadAndGetRagDocument(
LocalInputSource inputSource,
BaseRagDocumentUploadParameters<TAnnotationResponse> parameters,
PollingOptions pollingOptions
) throws IOException, InterruptedException {
if (pollingOptions == null) {
pollingOptions = PollingOptions.builder().build();
}
TAnnotationResponse initialResponse = uploadRagDocument(inputSource, parameters);
if (!"Processing".equals(initialResponse.getStatus())) {
return initialResponse;
}
return pollForRagDocument(parameters.getResponseClass(), initialResponse, pollingOptions);
Comment on lines +330 to +334
}

/**
* Not recommended for general use, prefer {@link #getReadyRagDocument}.
* You will need to poll until the document is ready for use.
* Get a document's info and annotations from the RAG database.
*
* @param responseClass The class of the response.
* @param documentId The document's ID.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse getRagDocument(
Class<TAnnotationResponse> responseClass,
String documentId
) {
if (documentId == null || documentId.isBlank()) {
throw new IllegalArgumentException("documentId must not be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Getting RAG document ID: {0}", documentId);
return mindeeApi.reqGetRagAnnotation(responseClass, documentId);
}
Comment thread
ianardee marked this conversation as resolved.

/**
* Get a document's info and annotations from the RAG database, polling if it is still processing.
*
* @param responseClass The class of the response.
* @param documentId The document's ID.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse getReadyRagDocument(
Class<TAnnotationResponse> responseClass,
String documentId
) throws InterruptedException {
return getReadyRagDocument(responseClass, documentId, null);
}

/**
* Get a document's info and annotations from the RAG database, polling if it is still processing.
*
* @param responseClass The class of the response.
* @param documentId The document's ID.
* @param pollingOptions Polling options (if null, default options are used).
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse getReadyRagDocument(
Class<TAnnotationResponse> responseClass,
String documentId,
PollingOptions pollingOptions
) throws InterruptedException {
TAnnotationResponse initialResponse = getRagDocument(responseClass, documentId);
if (!"Processing".equals(initialResponse.getStatus())) {
return initialResponse;
}

if (pollingOptions == null) {
pollingOptions = PollingOptions.builder().build();
}
return pollForRagDocument(responseClass, initialResponse, pollingOptions);
}

/**
* Not recommended for general use, prefer {@link #updateAndGetRagAnnotation}.
* You will need to poll until the document is ready for use.
* Update a document's annotations in the RAG database.
*
* @param parameters Annotation parameters.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse updateRagAnnotation(
BaseAnnotationParameters<TAnnotationResponse> parameters
) {
logger
.log(System.Logger.Level.INFO, "Updating RAG document ID: {0}", parameters.getDocumentId());
return mindeeApi.reqPatchRagAnnotation(parameters);
}

/**
* Update a document's annotations in the RAG database and poll until complete.
*
* @param parameters Annotation parameters.
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse updateAndGetRagAnnotation(
BaseAnnotationParameters<TAnnotationResponse> parameters
) throws InterruptedException {
return updateAndGetRagAnnotation(parameters, null);
}

/**
* Update a document's annotations in the RAG database and poll until complete.
*
* @param parameters Annotation parameters.
* @param pollingOptions Polling options (if null, default options are used).
* @return an instance of {@link BaseRagAnnotationResponse}.
*/
public <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse updateAndGetRagAnnotation(
BaseAnnotationParameters<TAnnotationResponse> parameters,
PollingOptions pollingOptions
) throws InterruptedException {
TAnnotationResponse initialResponse = updateRagAnnotation(parameters);
if (!"Processing".equals(initialResponse.getStatus())) {
return initialResponse;
}

if (pollingOptions == null) {
pollingOptions = PollingOptions.builder().build();
}
return pollForRagDocument(parameters.getResponseClass(), initialResponse, pollingOptions);
}

/**
* Delete a document from the RAG database.
* For extraction models only.
*
* @param documentId The document's ID.
* @return true if successful.
*/
public boolean deleteExtractionRagDocument(String documentId) {
if (documentId == null || documentId.isBlank()) {
throw new IllegalArgumentException("documentId must not be null or blank.");
}
logger.log(System.Logger.Level.INFO, "Deleting RAG document ID: {0}", documentId);
return mindeeApi.reqDeleteExtractionRagDocument(documentId);
}

/**
* Poll until the RAG document is finished processing or the max number of attempts is reached.
*
* @param responseClass The class of the response.
* @param initialResponse The initial annotation response.
* @param pollingOptions Polling options.
* @return an instance of {@link BaseRagAnnotationResponse}.
* @throws InterruptedException Throws if the thread is interrupted.
*/
private <TAnnotationResponse extends BaseRagAnnotationResponse> TAnnotationResponse pollForRagDocument(
Class<TAnnotationResponse> responseClass,
BaseRagAnnotationResponse initialResponse,
PollingOptions pollingOptions
) throws InterruptedException {
logger
.log(System.Logger.Level.INFO, "Polling for RAG document ID: {0}", initialResponse.getId());
int maxRetries = pollingOptions.getMaxRetries() + 1;

logger
.log(
System.Logger.Level.DEBUG,
"Waiting {0} seconds before attempting to retrieve the result...",
pollingOptions.getInitialDelaySec()
);

interruptibleSleep(
(long) (pollingOptions.getInitialDelaySec() * 1000),
pollingOptions.getCancelToken()
);

String documentId = initialResponse.getId();
long intervalMillis = (long) (pollingOptions.getIntervalSec() * 1000);
int retryCount = 1;

while (retryCount < maxRetries) {
logger.log(System.Logger.Level.DEBUG, "Poll attempt {0} of {1}", retryCount, maxRetries);

TAnnotationResponse response = getRagDocument(responseClass, documentId);
retryCount++;

String status = response.getStatus();
if ("Processing".equals(status)) {
interruptibleSleep(intervalMillis, pollingOptions.getCancelToken());
} else if ("Failed".equals(status)) {
throw new MindeeException("RAG failed without an error payload.");
} else {
return response;
}
}
throw new MindeeException("RAG polling not complete after " + retryCount + " attempts.");
}

/**
* Common logic for polling an asynchronous job for local & url files.
*
Expand Down
Loading
Loading