Script for migrating deleted Documents
This commit is contained in:
nisha
@@ -1,7 +1,4 @@
|
||||
package net.gepafin.tendermanagement.service;
|
||||
import net.gepafin.tendermanagement.entities.DocumentEntity;
|
||||
import net.gepafin.tendermanagement.entities.UserCompanyDelegationEntity;
|
||||
import net.gepafin.tendermanagement.enums.DocOtherSourceTypeEnum;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@@ -20,14 +17,6 @@ AmazonS3Service {
|
||||
|
||||
InputStream getFile(String s3Folder, String filePath) throws IOException;
|
||||
|
||||
String generateS3PathForDeletedDocument(DocOtherSourceTypeEnum typeOfDocument, Long callId, Long applicationId,Long amendmentId);
|
||||
|
||||
String generateS3PathForDeletedDocumentForOther();
|
||||
|
||||
void moveFile(String bucketName, String oldPath, String newPath);
|
||||
|
||||
void deleteDelegationfromS3(UserCompanyDelegationEntity userCompanyDelegationEntity);
|
||||
|
||||
void deleteFileFromS3(DocumentEntity documentEntity, Long callId, Long applicationId,Long amendmentId);
|
||||
UploadFileOnAmazonS3Response moveFile(String fileName, String oldPath, String newPath);
|
||||
|
||||
}
|
||||
@@ -61,9 +61,9 @@ public class AmazonS3ServiceImpl implements AmazonS3Service {
|
||||
@Value("${aws.s3.region}")
|
||||
private String region;
|
||||
|
||||
private String getBucketUrlPrefix() {
|
||||
return "https://" + bucketName + ".s3." + region + ".amazonaws.com/";
|
||||
}
|
||||
@Autowired
|
||||
S3ReUploadMigrationService s3ReUploadMigrationService;
|
||||
|
||||
|
||||
private String upload(String fileName, String s3Folder,
|
||||
MultipartFile file) throws IOException {
|
||||
@@ -146,32 +146,10 @@ public class AmazonS3ServiceImpl implements AmazonS3Service {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateS3PathForDeletedDocument(DocOtherSourceTypeEnum typeOfDocument, Long callId, Long applicationId,Long amendmentId) {
|
||||
try {
|
||||
return s3ConfigBean.generateDocumentPathForOther(typeOfDocument, callId, applicationId,amendmentId);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new CustomValidationException(
|
||||
Status.VALIDATION_ERROR,
|
||||
Translator.toLocale(GepafinConstant.S3_PATH_GENERATION_ERROR_MSG)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateS3PathForDeletedDocumentForOther() {
|
||||
try {
|
||||
return s3ConfigBean.generateDocumentPathForDelegationAndSignedDocument(DocOtherSourceTypeEnum.DELETED_USER_DELEGATION);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new CustomValidationException(
|
||||
Status.VALIDATION_ERROR,
|
||||
Translator.toLocale(GepafinConstant.S3_PATH_GENERATION_ERROR_MSG)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void moveFile(String bucketName, String oldPath, String newPath) {
|
||||
public UploadFileOnAmazonS3Response moveFile(String fileName, String oldPath, String newPath) {
|
||||
try {
|
||||
newPath = cleanNewPath(oldPath, newPath);
|
||||
oldPath = cleanOldPath(oldPath);
|
||||
log.info("Moving file from {} to {} in bucket {}", oldPath, newPath, bucketName);
|
||||
|
||||
CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, oldPath, bucketName, newPath);
|
||||
@@ -180,6 +158,8 @@ public class AmazonS3ServiceImpl implements AmazonS3Service {
|
||||
|
||||
s3Client.deleteObject(bucketName, oldPath);
|
||||
log.info("Original file deleted successfully: {}", oldPath);
|
||||
String filePath = s3Url + newPath;
|
||||
return UploadFileOnAmazonS3Response.builder().fileName(fileName).filePath(filePath).build();
|
||||
} catch (AmazonServiceException e) {
|
||||
log.error("AWS service error while moving file: {}", e.getErrorMessage(), e);
|
||||
throw e;
|
||||
@@ -192,36 +172,11 @@ public class AmazonS3ServiceImpl implements AmazonS3Service {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDelegationfromS3(UserCompanyDelegationEntity userCompanyDelegationEntity) {
|
||||
String oldS3Path = userCompanyDelegationEntity.getFilePath();
|
||||
String newS3Path = generateS3PathForDeletedDocumentForOther()
|
||||
+ "/" + oldS3Path.substring(oldS3Path.lastIndexOf("/") + 1);
|
||||
|
||||
String bucketUrlPrefix = getBucketUrlPrefix();
|
||||
if (oldS3Path.startsWith(bucketUrlPrefix)) {
|
||||
oldS3Path = oldS3Path.replace(bucketUrlPrefix, "");
|
||||
}
|
||||
moveFile(bucketName, oldS3Path, newS3Path);
|
||||
log.info("File for company ID {} successfully moved to deleted folder.", userCompanyDelegationEntity.getId());
|
||||
private String cleanNewPath(String oldPath, String newPath) {
|
||||
return newPath + "/" + oldPath.substring(oldPath.lastIndexOf("/") + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFileFromS3(DocumentEntity documentEntity, Long callId, Long applicationId,Long amendmentId) {
|
||||
try {
|
||||
String oldS3Path = documentEntity.getFilePath();
|
||||
String newS3Path = generateS3PathForDeletedDocument(DocOtherSourceTypeEnum.valueOf("DELETED_" + documentEntity.getSource().toUpperCase()), callId, applicationId,amendmentId)
|
||||
+ "/" + oldS3Path.substring(oldS3Path.lastIndexOf("/") + 1);
|
||||
String bucketUrlPrefix = getBucketUrlPrefix();
|
||||
if (oldS3Path.startsWith(bucketUrlPrefix)) {
|
||||
oldS3Path = oldS3Path.replace(bucketUrlPrefix, "");
|
||||
}
|
||||
|
||||
moveFile(bucketName, oldS3Path, newS3Path);
|
||||
log.info("File for document ID {} successfully moved to deleted folder.", documentEntity.getId());
|
||||
} catch (Exception e) {
|
||||
log.error("Error moving file for document ID {} to deleted folder: {}", documentEntity.getId(), e.getMessage());
|
||||
throw new CustomValidationException(Status.VALIDATION_ERROR, "Error occurred while moving file to deleted folder.");
|
||||
}
|
||||
private String cleanOldPath(String oldPath) {
|
||||
return oldPath.replace(s3Url, "");
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,17 @@ import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.gepafin.tendermanagement.dao.DocumentDao;
|
||||
import net.gepafin.tendermanagement.dao.S3PathConfig;
|
||||
import net.gepafin.tendermanagement.entities.ApplicationEntity;
|
||||
import net.gepafin.tendermanagement.entities.DocumentEntity;
|
||||
import net.gepafin.tendermanagement.enums.DocumentSourceTypeEnum;
|
||||
import net.gepafin.tendermanagement.repositories.ApplicationAmendmentRequestRepository;
|
||||
import net.gepafin.tendermanagement.repositories.ApplicationRepository;
|
||||
import net.gepafin.tendermanagement.repositories.ApplicationSignedDocumentRepository;
|
||||
import net.gepafin.tendermanagement.repositories.DocumentRepository;
|
||||
import net.gepafin.tendermanagement.service.AmazonS3Service;
|
||||
import net.gepafin.tendermanagement.service.ApplicationService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -50,13 +55,35 @@ public class S3ReUploadMigrationService {
|
||||
@Autowired
|
||||
private AmazonS3 amazonS3;
|
||||
|
||||
@Autowired
|
||||
ApplicationService applicationService;
|
||||
|
||||
@Value("${aws.s3.url}")
|
||||
private String s3Url;
|
||||
|
||||
@Autowired
|
||||
AmazonS3Service amazonS3Service;
|
||||
|
||||
@Value("${aws.s3.bucket.name}")
|
||||
private String bucketName;
|
||||
|
||||
@Value("${aws.s3.region}")
|
||||
private String region;
|
||||
|
||||
@Autowired
|
||||
private ApplicationAmendmentRequestRepository applicationAmendmentRequestRepository;
|
||||
|
||||
@Autowired
|
||||
private DocumentDao documentDao;
|
||||
|
||||
|
||||
private boolean migrationCompleted = false;
|
||||
|
||||
public String reUploadAndMigrateDocuments(String providedKey) {
|
||||
|
||||
Long totalDocuments=0L;
|
||||
Long failedDocuments=0L;
|
||||
Long processDocuments=0L;
|
||||
if (migrationCompleted) {
|
||||
return "Migration already completed.";
|
||||
}
|
||||
@@ -66,26 +93,48 @@ public class S3ReUploadMigrationService {
|
||||
return "Invalid or missing migration key.";
|
||||
}
|
||||
|
||||
List<DocumentEntity> documents = documentRepository.findAllByIsDeleteFalse();
|
||||
List<DocumentEntity> documents = documentRepository.findAllByIsDeleteTrue();
|
||||
totalDocuments = Long.valueOf(documents.size());
|
||||
|
||||
if (documents.isEmpty()) {
|
||||
return "No documents found to migrate.";
|
||||
}
|
||||
|
||||
for (DocumentEntity document : documents) {
|
||||
String oldUrl = document.getFilePath(); // This should contain the full URL
|
||||
log.info("Processing {}", oldUrl);
|
||||
|
||||
log.info("Processing for the Document id and url:{} ",document.getId(),document.getFilePath());
|
||||
try {
|
||||
File localFile = downloadFileFromS3(oldUrl);
|
||||
String newKey = generateNewS3Path(document); // Make sure this generates the correct new path
|
||||
String uploadedPath = uploadFileToNewBucket(localFile, newKey);
|
||||
updateDocumentPathAndNameEntry(document, uploadedPath);
|
||||
|
||||
Long callId = null;
|
||||
Long applicationId = null;
|
||||
Long amendmentId = null;
|
||||
|
||||
if (DocumentSourceTypeEnum.CALL.getValue().equalsIgnoreCase(document.getSource())) {
|
||||
callId = document.getSourceId();
|
||||
} else if (DocumentSourceTypeEnum.APPLICATION.getValue().equalsIgnoreCase(document.getSource())) {
|
||||
applicationId = document.getSourceId();
|
||||
ApplicationEntity applicationEntity = applicationService.validateApplication(applicationId);
|
||||
callId = applicationEntity.getCall().getId();
|
||||
}
|
||||
else if(DocumentSourceTypeEnum.AMENDMENT.getValue().equalsIgnoreCase(document.getSource())){
|
||||
amendmentId = document.getSourceId();
|
||||
ApplicationEntity applicationEntity = applicationAmendmentRequestRepository.findApplicationByAmendmentId(amendmentId);
|
||||
applicationId = applicationEntity.getId();
|
||||
callId = applicationEntity.getCall().getId();
|
||||
}
|
||||
|
||||
documentDao.deleteFileFromS3(document,callId,applicationId,amendmentId);
|
||||
processDocuments++;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Error processing document {}: {}", document.getId(), e.getMessage());
|
||||
failedDocuments++;
|
||||
}
|
||||
}
|
||||
log.info("Total Documents Fetched ",totalDocuments);
|
||||
log.info("Total Process Documents :{}",processDocuments);
|
||||
log.info("Total Failed Documents :{}",failedDocuments);
|
||||
return "Migrated Successfully.";
|
||||
|
||||
}
|
||||
|
||||
private boolean isValidKey(String providedKey) {
|
||||
@@ -177,13 +226,4 @@ public class S3ReUploadMigrationService {
|
||||
return parts[parts.length - 1];
|
||||
}
|
||||
|
||||
|
||||
private void updateDocumentPathAndNameEntry(DocumentEntity document, String newPath) {
|
||||
|
||||
String fileName = extractFileName(newPath);
|
||||
document.setFilePath(newPath);
|
||||
document.setFileName(fileName);
|
||||
documentRepository.save(document);
|
||||
log.info("Migrated document ID: {} to new path: {}", document.getId(), newPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.gepafin.tendermanagement.dao.DelegationDao;
|
||||
import net.gepafin.tendermanagement.dao.S3PathConfig;
|
||||
import net.gepafin.tendermanagement.entities.ApplicationSignedDocumentEntity;
|
||||
import net.gepafin.tendermanagement.entities.UserCompanyDelegationEntity;
|
||||
@@ -56,10 +57,16 @@ public class UserSignedAndDelegationServiceImpl {
|
||||
@Value("${aws.s3.url}")
|
||||
private String s3Url;
|
||||
|
||||
@Autowired
|
||||
private DelegationDao delegationDao;
|
||||
|
||||
private boolean migrationCompleted = false;
|
||||
|
||||
public String migrateUserDelegatedDocuments(String providedKey) {
|
||||
|
||||
Long totalDocuments=0L;
|
||||
Long failedDocuments=0L;
|
||||
Long processDocuments=0L;
|
||||
if (migrationCompleted) {
|
||||
return "Migration already completed.";
|
||||
}
|
||||
@@ -69,25 +76,27 @@ public class UserSignedAndDelegationServiceImpl {
|
||||
return "Invalid or missing migration key.";
|
||||
}
|
||||
|
||||
List<UserCompanyDelegationEntity> documents = userCompanyDelegationRepository.findAllByStatus("ACTIVE");
|
||||
|
||||
List<UserCompanyDelegationEntity> documents = userCompanyDelegationRepository.findAllByStatus("INACTIVE");
|
||||
totalDocuments = Long.valueOf(documents.size());
|
||||
if (documents.isEmpty()) {
|
||||
return "No documents found to migrate.";
|
||||
}
|
||||
|
||||
for (UserCompanyDelegationEntity document : documents) {
|
||||
String oldUrl = document.getFilePath();
|
||||
log.info("Processing user designated document: {}", oldUrl);
|
||||
log.info("Processing user designated document and old Url: {}", document.getId(),oldUrl);
|
||||
|
||||
try {
|
||||
File localFile = downloadFileFromS3(oldUrl);
|
||||
String newKey = generateNewS3PathForDelegationDoc();
|
||||
String uploadedPath = uploadFileToNewBucket(localFile, newKey);
|
||||
updateDelegatedDocumentPathAndNameEntry(document, uploadedPath);
|
||||
delegationDao.deleteDelegationFromS3(document);
|
||||
processDocuments++;
|
||||
} catch (Exception e) {
|
||||
log.error("Error processing user designated document {}: {}", document.getId(), e.getMessage());
|
||||
failedDocuments++;
|
||||
}
|
||||
}
|
||||
log.info("Total Documents Fetched:{} ",totalDocuments);
|
||||
log.info("Total Process Documents :{}",processDocuments);
|
||||
log.info("Total Failed Documents :{}",failedDocuments);
|
||||
return "Migrated";
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user